Something went wrong. Try again.
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625// Byte lexer over UTF-8 input in linear memory. Positions are stream offsets// (byte offsets from the start of the whole input); a byte lives at offset +// delta. The lexer is streaming: when a token touches the end of the buffered// input before `fin`, it sets `more`, leaves `pos` at the token's pre-trivia// position and produces nothing, so the same scan can be retried after the// host pushes more bytes. The parser drives the context-dependent rescans// (regex, template continuation, `>` compounds) because it knows which state// it is in.import { Memory } from 'zena:memory';import { keywordAt, tokenTypeIn, PUNCT, PUNCT_ENT, LEXC, LEXC_MASK } from './keywords.zena';import { CLASS, ID_START, ID_CHAR, DIGIT, WS, NL } from './chartable.zena';
// The first address in [a, b) holding byte x, y or z, else b, 16 bytes per step (engine/rt.wat). It reads up// to 15 bytes past b, which the input buffer's 16 bytes of slack keep in bounds.@external("rt", "find3")declare function find3(a: i32, b: i32, x: i32, y: i32, z: i32): i32;// The first address p in [a, b) with `*/` at p, else b (engine/rt.wat). Reads up to 16 bytes past b.@external("rt", "findClose")declare function findClose(a: i32, b: i32): i32;
export var delta: i32 = 0;export var inEnd: i32 = 0;export var fin: i32 = 0;export var pos: i32 = 0;export var more: i32 = 0;
export var tb: i32 = 0; // token begin, including leading triviaexport var ts: i32 = 0; // token startexport var te: i32 = 0; // token endexport var tt: i32 = 0; // token typeexport var tk: i32 = 0; // keyword id when tt == T_NAMEexport var tnl: i32 = 0; // 1 when a line terminator precedes the token
export let T_EOF: i32 = 0;export let T_NAME: i32 = 1;export let T_NUM: i32 = 2;export let T_STR: i32 = 3;export let T_TEMPLATE: i32 = 4;export let T_TEMPLATE_HEAD: i32 = 5;export let T_TEMPLATE_MID: i32 = 6;export let T_TEMPLATE_TAIL: i32 = 7;export let T_REGEX: i32 = 8;export let T_PRIVATE: i32 = 9;export let T_LBRACE: i32 = 10;export let T_RBRACE: i32 = 11;export let T_LPAREN: i32 = 12;export let T_RPAREN: i32 = 13;export let T_LBRACK: i32 = 14;export let T_RBRACK: i32 = 15;export let T_DOT: i32 = 16;export let T_ELLIPSIS: i32 = 17;export let T_SEMI: i32 = 18;export let T_COMMA: i32 = 19;export let T_COLON: i32 = 20;export let T_QUESTION: i32 = 21;export let T_QDOT: i32 = 22;export let T_ARROW: i32 = 23;export let T_AT: i32 = 24;export let T_BANG: i32 = 25;export let T_TILDE: i32 = 26;export let T_INC: i32 = 27;export let T_DEC: i32 = 28;// Binary operators, T_QQ..T_STARSTAR; precOf gives their precedence.export let T_QQ: i32 = 30;export let T_BARBAR: i32 = 31;export let T_AMPAMP: i32 = 32;export let T_BAR: i32 = 33;export let T_CARET: i32 = 34;export let T_AMP: i32 = 35;export let T_EQEQ: i32 = 36;export let T_NEQ: i32 = 37;export let T_EQEQEQ: i32 = 38;export let T_NEQEQ: i32 = 39;export let T_LT: i32 = 40;export let T_GT: i32 = 41;export let T_LTE: i32 = 42;export let T_GTE: i32 = 43;export let T_SHL: i32 = 44;export let T_SHR: i32 = 45;export let T_USHR: i32 = 46;export let T_PLUS: i32 = 47;export let T_MINUS: i32 = 48;export let T_STAR: i32 = 49;export let T_SLASH: i32 = 50;export let T_PERCENT: i32 = 51;export let T_STARSTAR: i32 = 52;// Assignment operators, T_EQ..T_QQ_EQ.export let T_EQ: i32 = 60;export let T_PLUS_EQ: i32 = 61;export let T_MINUS_EQ: i32 = 62;export let T_STAR_EQ: i32 = 63;export let T_SLASH_EQ: i32 = 64;export let T_PERCENT_EQ: i32 = 65;export let T_STARSTAR_EQ: i32 = 66;export let T_SHL_EQ: i32 = 67;export let T_SHR_EQ: i32 = 68;export let T_USHR_EQ: i32 = 69;export let T_AMP_EQ: i32 = 70;export let T_BAR_EQ: i32 = 71;export let T_CARET_EQ: i32 = 72;export let T_AMPAMP_EQ: i32 = 73;export let T_BARBAR_EQ: i32 = 74;export let T_QQ_EQ: i32 = 75;
// Whether the current token's type is in the set mask s (a TS_ set from keywords.zena).export function tokenIn(s: i64): boolean { return tokenTypeIn(tt, s);}
export function isBinop(t: i32): boolean { return t >= T_QQ && t <= T_STARSTAR;}
export function isAssign(t: i32): boolean { return t >= T_EQ && t <= T_QQ_EQ;}
// ts-blank-space's getBinaryOperatorPrecedence, for the binary-operator tokens.export function precOf(t: i32): i32 { if (t == T_STARSTAR) return 15; if (t >= T_STAR) return 14; if (t >= T_PLUS) return 13; if (t >= T_SHL) return 12; if (t >= T_LT) return 11; if (t >= T_EQEQ) return 10; if (t == T_AMP) return 9; if (t == T_CARET) return 8; if (t == T_BAR) return 7; if (t == T_AMPAMP) return 6; if (t == T_BARBAR) return 5; return 4;}
export function setInput(d: i32, end: i32, f: i32): void { delta = d; inEnd = end; fin = f;}
export function seek(p: i32): void { pos = p;}
export function clearMore(): void { more = 0;}
// Re-lexes the token that begins (trivia included) at p.export function reseek(p: i32): void { pos = p; scan();}
function needMore(start: i32): void { more = 1; pos = start;}
// Byte classification, from the 256-entry table in engine/chartable.zena: one load plus a mask instead of// a range-check chain. ID_CHAR is ID_START | DIGIT, so the hot identifier/number continuation loops below// stay a single mask test against one load.function isIdStart(c: i32): boolean { return (Memory.default.getU8(CLASS + c) & ID_START) != 0;}
function isIdChar(c: i32): boolean { return (Memory.default.getU8(CLASS + c) & ID_CHAR) != 0;}
function isDigit(c: i32): boolean { return (Memory.default.getU8(CLASS + c) & DIGIT) != 0;}
// A byte at stream offset p, or 0 past the buffered end.function at(p: i32): i32 { if (p >= inEnd) return 0; return Memory.default.getU8(p + delta);}
// The end of an identifier's `\uXXXX` or `\u{...}` escape at p (the end of input when unterminated).function escapeEnd(p: i32): i32 { var q = p + 2; if (at(p + 1) == 117 && at(q) == 123) { while (q < inEnd && at(q) != 125) q = q + 1; if (q < inEnd) q = q + 1; } return q;}
// Classifies a non-ASCII sequence at p: 0 not whitespace, 1 whitespace,// 2 line terminator, -1 not enough bytes buffered to tell.function uniSpace(p: i32): i32 { let c = at(p); if (c == 0xc2) { if (p + 1 >= inEnd) return -1; if (at(p + 1) == 0xa0) return 1; return 0; } if (c == 0xef || c == 0xe2 || c == 0xe3 || c == 0xe1) { if (p + 2 >= inEnd) return -1; let b1 = at(p + 1); let b2 = at(p + 2); if (c == 0xef) { if (b1 == 0xbb && b2 == 0xbf) return 1; return 0; } if (c == 0xe2) { if (b1 == 0x80) { if (b2 >= 0x80 && b2 <= 0x8a) return 1; if (b2 == 0xa8 || b2 == 0xa9) return 2; if (b2 == 0xaf) return 1; return 0; } if (b1 == 0x81 && b2 == 0x9f) return 1; return 0; } if (c == 0xe3) { if (b1 == 0x80 && b2 == 0x80) return 1; return 0; } if (b1 == 0x9a && b2 == 0x80) return 1; return 0; } return 0;}
// Finishes a token of type t ending at e, or asks for more input when it touches the buffered end.function finish(t: i32, e: i32, start: i32): void { if (e + 3 > inEnd && fin == 0) { needMore(start); return; } tt = t; te = e; pos = e;}
// Token replay cache. Lookahead ends by reseeking to a mark, so most tokens are lexed again; scan// replays them from a direct-mapped table instead. Entry at LEXC + (tb & LEXC_MASK) * 16 holds// [tb, ts, te, tt | tk << 8 | tnl << 16] for scan's own result at tb (never a rescan's, never a// needMore). Scan is a pure function of the bytes at stream offsets, and a token it completes// before `fin` never depends on bytes pushed later (it is at least 3 bytes short of the buffered// end), so an entry stays valid for the whole stream: only init's new stream clears it.export function lexReset(): void { var i = 0; while (i <= LEXC_MASK) { Memory.default.setI32(LEXC + (i << 4), -1); i = i + 1; }}
export function scan(): void { let start = pos; let c = LEXC + ((start & LEXC_MASK) << 4); if (Memory.default.getI32(c) == start) { tb = start; ts = Memory.default.getI32(c + 4); let e = Memory.default.getI32(c + 8); let k = Memory.default.getI32(c + 12); te = e; pos = e; tt = k & 255; tk = (k >> 8) & 255; tnl = k >> 16; return; } let m = more; more = 0; lexToken(); if (more == 0) { Memory.default.setI32(c, start); Memory.default.setI32(c + 4, ts); Memory.default.setI32(c + 8, te); Memory.default.setI32(c + 12, tt | (tk << 8) | (tnl << 16)); more = m; }}
function lexToken(): void { let start = pos; tb = start; var p = start; var nl = 0; let d = delta; let end = inEnd; tk = 0; while (true) { if (p >= end) { if (fin == 0) { needMore(start); return; } break; } let c = Memory.default.getU8(p + d); let cls = Memory.default.getU8(CLASS + c); if ((cls & WS) != 0) { p = p + 1; } else if ((cls & NL) != 0) { nl = 1; p = p + 1; } else if (c == 47) { if (p + 1 >= end) { if (fin == 0) { needMore(start); return; } break; } let c2 = Memory.default.getU8(p + 1 + d); if (c2 == 47) { p = p + 2; while (p < end) { p = find3(p + d, end + d, 10, 13, 0xe2) - d; if (p >= end) break; if (Memory.default.getU8(p + d) != 0xe2 || uniSpace(p) == 2) break; p = p + 1; } if (p >= end && fin == 0) { needMore(start); return; } } else if (c2 == 42) { p = p + 2; var closed = 0; while (p + 1 < end) { // Until a line terminator is seen, stops at `*`, \n and \r; after one, only at `*/`. A `*` at the // chunk's last byte is never a match, so the comment stays open and the scan retries with more input. if (nl != 0) { p = findClose(p + d, end - 1 + d) - d; if (p + 1 >= end) break; closed = 1; p = p + 2; break; } p = find3(p + d, end - 1 + d, 42, 10, 13) - d; if (p + 1 >= end) break; if (Memory.default.getU8(p + d) != 42) { nl = 1; } else if (Memory.default.getU8(p + 1 + d) == 47) { closed = 1; p = p + 2; break; } p = p + 1; } if (closed == 0) { if (fin == 0) { needMore(start); return; } p = end; } } else { break; } } else if (c == 35 && p == 0 && at(1) == 33) { // #! shebang while (p < end) { let x = Memory.default.getU8(p + d); if (x == 10 || x == 13) break; p = p + 1; } } else if (c >= 0x80) { let k = uniSpace(p); if (k < 0) { if (fin == 0) { needMore(start); return; } break; } if (k == 0) break; if (k == 2) nl = 1; if (c == 0xc2) p = p + 2; else p = p + 3; } else { break; } } ts = p; tnl = nl; if (p >= end) { tt = T_EOF; te = p; pos = p; return; } let c = Memory.default.getU8(p + d);
if (isIdStart(c) || c >= 0x80 || c == 92) { var q = p + 1; if (c == 92) q = escapeEnd(p); while (q < end) { while (q < end && isIdChar(Memory.default.getU8(q + d))) q = q + 1; if (q >= end) break; let x = Memory.default.getU8(q + d); if (x == 92) { q = escapeEnd(q); } else if (x >= 0x80) { let k = uniSpace(q); if (k < 0) { needMore(start); return; } if (k != 0) break; q = q + 1; } else { break; } } if (q + 3 > end && fin == 0) { needMore(start); return; } let n = q - p; // The buffer always has 16 bytes of slack, so keywordAt's wide loads stay in bounds. if (c >= 97 && c <= 122) tk = keywordAt(p + d, n); tt = T_NAME; te = q; pos = q; return; }
if (isDigit(c) || (c == 46 && isDigit(at(p + 1)))) { var q = p + 1; var seenDot = c == 46; var radix = false; if (c == 48) { let x = at(p + 1); if (x == 120 || x == 88 || x == 111 || x == 79 || x == 98 || x == 66) { radix = true; q = p + 2; } } while (q < end) { let x = Memory.default.getU8(q + d); if (isIdChar(x)) { q = q + 1; if (!radix && (x == 101 || x == 69)) { let s = at(q); if (s == 43 || s == 45) q = q + 1; } } else if (x == 46 && !seenDot && !radix) { seenDot = true; q = q + 1; } else { break; } } finish(T_NUM, q, start); return; }
if (c == 34 || c == 39) { var q = p + 1; while (q < end) { q = find3(q + d, end + d, c, 92, c) - d; if (q >= end) break; if (Memory.default.getU8(q + d) == c) { finish(T_STR, q + 1, start); return; } q = q + 2; } if (fin == 0) { needMore(start); return; } finish(T_STR, end, start); return; }
if (c == 96) { scanTemplate(p + 1, T_TEMPLATE, T_TEMPLATE_HEAD, start); return; }
if (c == 35) { var q = p + 1; while (q < end) { let x = Memory.default.getU8(q + d); if (isIdChar(x) || x >= 0x80) q = q + 1; else break; } finish(T_PRIVATE, q, start); return; }
let c1 = at(p + 1); if (c == 46) { if (c1 == 46 && at(p + 2) == 46) { finish(T_ELLIPSIS, p + 3, start); return; } finish(T_DOT, p + 1, start); return; } if (c == 63 && c1 == 46 && !isDigit(at(p + 2))) { finish(T_QDOT, p + 2, start); return; } if (c == 61 && c1 == 62) { finish(T_ARROW, p + 2, start); return; } // Every other punctuator from PUNCT's entry for c (engine/keywords.wat): the byte alone, doubled, or with `=`, // then a further `=` where that forms a longer one. A byte with no entry (a stray control byte) is a // one-byte token. `>` is always a lone `>`; the expression parser widens it with rescanGreater. let e = PUNCT_ENT + Memory.default.getU8(PUNCT + c) * 5; var q = p + 1; var t = Memory.default.getU8(e); var x = 0; if (c1 == c && Memory.default.getU8(e + 3) != 0) { q = p + 2; t = Memory.default.getU8(e + 3); x = Memory.default.getU8(e + 4); } else if (c1 == 61 && Memory.default.getU8(e + 1) != 0) { q = p + 2; t = Memory.default.getU8(e + 1); x = Memory.default.getU8(e + 2); } if (x != 0 && at(q) == 61) { t = x; q = q + 1; } finish(t, q, start);}
// Scans template text from q to a closing backtick (type whole) or `${` (type head).function scanTemplate(q0: i32, whole: i32, head: i32, start: i32): void { let d = delta; let end = inEnd; var q = q0; while (q < end) { q = find3(q + d, end + d, 96, 36, 92) - d; if (q >= end) break; let x = Memory.default.getU8(q + d); if (x == 96) { finish(whole, q + 1, start); return; } if (x == 36) { if (q + 1 >= end) break; if (Memory.default.getU8(q + 1 + d) == 123) { finish(head, q + 2, start); return; } q = q + 1; } else { q = q + 2; } } if (fin == 0) { needMore(start); return; } finish(whole, end, start);}
// The current token is `}` closing a template substitution: rescan it as the next template part.export function rescanTemplate(): void { scanTemplate(ts + 1, T_TEMPLATE_TAIL, T_TEMPLATE_MID, tb);}
// The current token is `/` or `/=` in operand position: rescan it as a regular expression.export function rescanRegex(): void { let d = delta; let end = inEnd; var q = ts + 1; var inClass = false; var closed = false; while (q < end) { let x = Memory.default.getU8(q + d); if (x == 10 || x == 13) break; if (x == 92) { q = q + 2; } else if (x == 91) { inClass = true; q = q + 1; } else if (x == 93) { inClass = false; q = q + 1; } else if (x == 47 && !inClass) { q = q + 1; closed = true; break; } else { q = q + 1; } } if (!closed && fin == 0) { needMore(tb); return; } while (q < end) { let x = Memory.default.getU8(q + d); if (isIdChar(x)) q = q + 1; else break; } finish(T_REGEX, q, tb);}
// The current token is `<<` where type arguments may start: split off a lone `<`.export function rescanLess(): void { finish(T_LT, ts + 1, tb);}
// The current token is `>` in operator position: widen it to `>=`, `>>`, `>>>`, `>>=` or `>>>=`.export function rescanGreater(): void { var q = ts + 1; var t = T_GT; if (at(q) == 62) { q = q + 1; t = T_SHR; if (at(q) == 62) { q = q + 1; t = T_USHR; } if (at(q) == 61) { q = q + 1; if (t == T_SHR) t = T_SHR_EQ; else t = T_USHR_EQ; } } else if (at(q) == 61) { q = q + 1; t = T_GTE; } finish(t, q, tb);}