// Byte lexer over UTF-8 input in linear memory. Positions are stream offsets
// (byte offsets from the start of the whole input); a byte lives at offset +
// delta. The lexer is streaming: when a token touches the end of the buffered
// input before `fin`, it sets `more`, leaves `pos` at the token's pre-trivia
// position and produces nothing, so the same scan can be retried after the
// host pushes more bytes. The parser drives the context-dependent rescans
// (regex, template continuation, `>` compounds) because it knows which state
// it is in.
import { Memory } from 'zena:memory';
import { keywordAt, tokenTypeIn, PUNCT, PUNCT_ENT, LEXC, LEXC_MASK } from './keywords.zena';
import { CLASS, ID_START, ID_CHAR, DIGIT, WS, NL } from './chartable.zena';

// The first address in [a, b) holding byte x, y or z, else b, 16 bytes per step (engine/rt.wat). It reads up
// to 15 bytes past b, which the input buffer's 16 bytes of slack keep in bounds.
@external("rt", "find3")
declare function find3(a: i32, b: i32, x: i32, y: i32, z: i32): i32;
// The first address p in [a, b) with `*/` at p, else b (engine/rt.wat). Reads up to 16 bytes past b.
@external("rt", "findClose")
declare function findClose(a: i32, b: i32): i32;

export var delta: i32 = 0;
export var inEnd: i32 = 0;
export var fin: i32 = 0;
export var pos: i32 = 0;
export var more: i32 = 0;

export var tb: i32 = 0; // token begin, including leading trivia
export var ts: i32 = 0; // token start
export var te: i32 = 0; // token end
export var tt: i32 = 0; // token type
export var tk: i32 = 0; // keyword id when tt == T_NAME
export var tnl: i32 = 0; // 1 when a line terminator precedes the token

export let T_EOF: i32 = 0;
export let T_NAME: i32 = 1;
export let T_NUM: i32 = 2;
export let T_STR: i32 = 3;
export let T_TEMPLATE: i32 = 4;
export let T_TEMPLATE_HEAD: i32 = 5;
export let T_TEMPLATE_MID: i32 = 6;
export let T_TEMPLATE_TAIL: i32 = 7;
export let T_REGEX: i32 = 8;
export let T_PRIVATE: i32 = 9;
export let T_LBRACE: i32 = 10;
export let T_RBRACE: i32 = 11;
export let T_LPAREN: i32 = 12;
export let T_RPAREN: i32 = 13;
export let T_LBRACK: i32 = 14;
export let T_RBRACK: i32 = 15;
export let T_DOT: i32 = 16;
export let T_ELLIPSIS: i32 = 17;
export let T_SEMI: i32 = 18;
export let T_COMMA: i32 = 19;
export let T_COLON: i32 = 20;
export let T_QUESTION: i32 = 21;
export let T_QDOT: i32 = 22;
export let T_ARROW: i32 = 23;
export let T_AT: i32 = 24;
export let T_BANG: i32 = 25;
export let T_TILDE: i32 = 26;
export let T_INC: i32 = 27;
export let T_DEC: i32 = 28;
// Binary operators, T_QQ..T_STARSTAR; precOf gives their precedence.
export let T_QQ: i32 = 30;
export let T_BARBAR: i32 = 31;
export let T_AMPAMP: i32 = 32;
export let T_BAR: i32 = 33;
export let T_CARET: i32 = 34;
export let T_AMP: i32 = 35;
export let T_EQEQ: i32 = 36;
export let T_NEQ: i32 = 37;
export let T_EQEQEQ: i32 = 38;
export let T_NEQEQ: i32 = 39;
export let T_LT: i32 = 40;
export let T_GT: i32 = 41;
export let T_LTE: i32 = 42;
export let T_GTE: i32 = 43;
export let T_SHL: i32 = 44;
export let T_SHR: i32 = 45;
export let T_USHR: i32 = 46;
export let T_PLUS: i32 = 47;
export let T_MINUS: i32 = 48;
export let T_STAR: i32 = 49;
export let T_SLASH: i32 = 50;
export let T_PERCENT: i32 = 51;
export let T_STARSTAR: i32 = 52;
// Assignment operators, T_EQ..T_QQ_EQ.
export let T_EQ: i32 = 60;
export let T_PLUS_EQ: i32 = 61;
export let T_MINUS_EQ: i32 = 62;
export let T_STAR_EQ: i32 = 63;
export let T_SLASH_EQ: i32 = 64;
export let T_PERCENT_EQ: i32 = 65;
export let T_STARSTAR_EQ: i32 = 66;
export let T_SHL_EQ: i32 = 67;
export let T_SHR_EQ: i32 = 68;
export let T_USHR_EQ: i32 = 69;
export let T_AMP_EQ: i32 = 70;
export let T_BAR_EQ: i32 = 71;
export let T_CARET_EQ: i32 = 72;
export let T_AMPAMP_EQ: i32 = 73;
export let T_BARBAR_EQ: i32 = 74;
export let T_QQ_EQ: i32 = 75;

// Whether the current token's type is in the set mask s (a TS_ set from keywords.zena).
export function tokenIn(s: i64): boolean {
  return tokenTypeIn(tt, s);
}

export function isBinop(t: i32): boolean {
  return t >= T_QQ && t <= T_STARSTAR;
}

export function isAssign(t: i32): boolean {
  return t >= T_EQ && t <= T_QQ_EQ;
}

// ts-blank-space's getBinaryOperatorPrecedence, for the binary-operator tokens.
export function precOf(t: i32): i32 {
  if (t == T_STARSTAR) return 15;
  if (t >= T_STAR) return 14;
  if (t >= T_PLUS) return 13;
  if (t >= T_SHL) return 12;
  if (t >= T_LT) return 11;
  if (t >= T_EQEQ) return 10;
  if (t == T_AMP) return 9;
  if (t == T_CARET) return 8;
  if (t == T_BAR) return 7;
  if (t == T_AMPAMP) return 6;
  if (t == T_BARBAR) return 5;
  return 4;
}

export function setInput(d: i32, end: i32, f: i32): void {
  delta = d;
  inEnd = end;
  fin = f;
}

export function seek(p: i32): void {
  pos = p;
}

export function clearMore(): void {
  more = 0;
}

// Re-lexes the token that begins (trivia included) at p.
export function reseek(p: i32): void {
  pos = p;
  scan();
}

function needMore(start: i32): void {
  more = 1;
  pos = start;
}

// Byte classification, from the 256-entry table in engine/chartable.zena: one load plus a mask instead of
// a range-check chain. ID_CHAR is ID_START | DIGIT, so the hot identifier/number continuation loops below
// stay a single mask test against one load.
function isIdStart(c: i32): boolean {
  return (Memory.default.getU8(CLASS + c) & ID_START) != 0;
}

function isIdChar(c: i32): boolean {
  return (Memory.default.getU8(CLASS + c) & ID_CHAR) != 0;
}

function isDigit(c: i32): boolean {
  return (Memory.default.getU8(CLASS + c) & DIGIT) != 0;
}

// A byte at stream offset p, or 0 past the buffered end.
function at(p: i32): i32 {
  if (p >= inEnd) return 0;
  return Memory.default.getU8(p + delta);
}

// The end of an identifier's `\uXXXX` or `\u{...}` escape at p (the end of input when unterminated).
function escapeEnd(p: i32): i32 {
  var q = p + 2;
  if (at(p + 1) == 117 && at(q) == 123) {
    while (q < inEnd && at(q) != 125) q = q + 1;
    if (q < inEnd) q = q + 1;
  }
  return q;
}

// Classifies a non-ASCII sequence at p: 0 not whitespace, 1 whitespace,
// 2 line terminator, -1 not enough bytes buffered to tell.
function uniSpace(p: i32): i32 {
  let c = at(p);
  if (c == 0xc2) {
    if (p + 1 >= inEnd) return -1;
    if (at(p + 1) == 0xa0) return 1;
    return 0;
  }
  if (c == 0xef || c == 0xe2 || c == 0xe3 || c == 0xe1) {
    if (p + 2 >= inEnd) return -1;
    let b1 = at(p + 1);
    let b2 = at(p + 2);
    if (c == 0xef) {
      if (b1 == 0xbb && b2 == 0xbf) return 1;
      return 0;
    }
    if (c == 0xe2) {
      if (b1 == 0x80) {
        if (b2 >= 0x80 && b2 <= 0x8a) return 1;
        if (b2 == 0xa8 || b2 == 0xa9) return 2;
        if (b2 == 0xaf) return 1;
        return 0;
      }
      if (b1 == 0x81 && b2 == 0x9f) return 1;
      return 0;
    }
    if (c == 0xe3) {
      if (b1 == 0x80 && b2 == 0x80) return 1;
      return 0;
    }
    if (b1 == 0x9a && b2 == 0x80) return 1;
    return 0;
  }
  return 0;
}

// Finishes a token of type t ending at e, or asks for more input when it touches the buffered end.
function finish(t: i32, e: i32, start: i32): void {
  if (e + 3 > inEnd && fin == 0) {
    needMore(start);
    return;
  }
  tt = t;
  te = e;
  pos = e;
}

// Token replay cache. Lookahead ends by reseeking to a mark, so most tokens are lexed again; scan
// replays them from a direct-mapped table instead. Entry at LEXC + (tb & LEXC_MASK) * 16 holds
// [tb, ts, te, tt | tk << 8 | tnl << 16] for scan's own result at tb (never a rescan's, never a
// needMore). Scan is a pure function of the bytes at stream offsets, and a token it completes
// before `fin` never depends on bytes pushed later (it is at least 3 bytes short of the buffered
// end), so an entry stays valid for the whole stream: only init's new stream clears it.
export function lexReset(): void {
  var i = 0;
  while (i <= LEXC_MASK) {
    Memory.default.setI32(LEXC + (i << 4), -1);
    i = i + 1;
  }
}

export function scan(): void {
  let start = pos;
  let c = LEXC + ((start & LEXC_MASK) << 4);
  if (Memory.default.getI32(c) == start) {
    tb = start;
    ts = Memory.default.getI32(c + 4);
    let e = Memory.default.getI32(c + 8);
    let k = Memory.default.getI32(c + 12);
    te = e;
    pos = e;
    tt = k & 255;
    tk = (k >> 8) & 255;
    tnl = k >> 16;
    return;
  }
  let m = more;
  more = 0;
  lexToken();
  if (more == 0) {
    Memory.default.setI32(c, start);
    Memory.default.setI32(c + 4, ts);
    Memory.default.setI32(c + 8, te);
    Memory.default.setI32(c + 12, tt | (tk << 8) | (tnl << 16));
    more = m;
  }
}

function lexToken(): void {
  let start = pos;
  tb = start;
  var p = start;
  var nl = 0;
  let d = delta;
  let end = inEnd;
  tk = 0;
  while (true) {
    if (p >= end) {
      if (fin == 0) {
        needMore(start);
        return;
      }
      break;
    }
    let c = Memory.default.getU8(p + d);
    let cls = Memory.default.getU8(CLASS + c);
    if ((cls & WS) != 0) {
      p = p + 1;
    } else if ((cls & NL) != 0) {
      nl = 1;
      p = p + 1;
    } else if (c == 47) {
      if (p + 1 >= end) {
        if (fin == 0) {
          needMore(start);
          return;
        }
        break;
      }
      let c2 = Memory.default.getU8(p + 1 + d);
      if (c2 == 47) {
        p = p + 2;
        while (p < end) {
          p = find3(p + d, end + d, 10, 13, 0xe2) - d;
          if (p >= end) break;
          if (Memory.default.getU8(p + d) != 0xe2 || uniSpace(p) == 2) break;
          p = p + 1;
        }
        if (p >= end && fin == 0) {
          needMore(start);
          return;
        }
      } else if (c2 == 42) {
        p = p + 2;
        var closed = 0;
        while (p + 1 < end) {
          // Until a line terminator is seen, stops at `*`, \n and \r; after one, only at `*/`. A `*` at the
          // chunk's last byte is never a match, so the comment stays open and the scan retries with more input.
          if (nl != 0) {
            p = findClose(p + d, end - 1 + d) - d;
            if (p + 1 >= end) break;
            closed = 1;
            p = p + 2;
            break;
          }
          p = find3(p + d, end - 1 + d, 42, 10, 13) - d;
          if (p + 1 >= end) break;
          if (Memory.default.getU8(p + d) != 42) {
            nl = 1;
          } else if (Memory.default.getU8(p + 1 + d) == 47) {
            closed = 1;
            p = p + 2;
            break;
          }
          p = p + 1;
        }
        if (closed == 0) {
          if (fin == 0) {
            needMore(start);
            return;
          }
          p = end;
        }
      } else {
        break;
      }
    } else if (c == 35 && p == 0 && at(1) == 33) {
      // #! shebang
      while (p < end) {
        let x = Memory.default.getU8(p + d);
        if (x == 10 || x == 13) break;
        p = p + 1;
      }
    } else if (c >= 0x80) {
      let k = uniSpace(p);
      if (k < 0) {
        if (fin == 0) {
          needMore(start);
          return;
        }
        break;
      }
      if (k == 0) break;
      if (k == 2) nl = 1;
      if (c == 0xc2) p = p + 2;
      else p = p + 3;
    } else {
      break;
    }
  }
  ts = p;
  tnl = nl;
  if (p >= end) {
    tt = T_EOF;
    te = p;
    pos = p;
    return;
  }
  let c = Memory.default.getU8(p + d);

  if (isIdStart(c) || c >= 0x80 || c == 92) {
    var q = p + 1;
    if (c == 92) q = escapeEnd(p);
    while (q < end) {
      while (q < end && isIdChar(Memory.default.getU8(q + d))) q = q + 1;
      if (q >= end) break;
      let x = Memory.default.getU8(q + d);
      if (x == 92) {
        q = escapeEnd(q);
      } else if (x >= 0x80) {
        let k = uniSpace(q);
        if (k < 0) {
          needMore(start);
          return;
        }
        if (k != 0) break;
        q = q + 1;
      } else {
        break;
      }
    }
    if (q + 3 > end && fin == 0) {
      needMore(start);
      return;
    }
    let n = q - p;
    // The buffer always has 16 bytes of slack, so keywordAt's wide loads stay in bounds.
    if (c >= 97 && c <= 122) tk = keywordAt(p + d, n);
    tt = T_NAME;
    te = q;
    pos = q;
    return;
  }

  if (isDigit(c) || (c == 46 && isDigit(at(p + 1)))) {
    var q = p + 1;
    var seenDot = c == 46;
    var radix = false;
    if (c == 48) {
      let x = at(p + 1);
      if (x == 120 || x == 88 || x == 111 || x == 79 || x == 98 || x == 66) {
        radix = true;
        q = p + 2;
      }
    }
    while (q < end) {
      let x = Memory.default.getU8(q + d);
      if (isIdChar(x)) {
        q = q + 1;
        if (!radix && (x == 101 || x == 69)) {
          let s = at(q);
          if (s == 43 || s == 45) q = q + 1;
        }
      } else if (x == 46 && !seenDot && !radix) {
        seenDot = true;
        q = q + 1;
      } else {
        break;
      }
    }
    finish(T_NUM, q, start);
    return;
  }

  if (c == 34 || c == 39) {
    var q = p + 1;
    while (q < end) {
      q = find3(q + d, end + d, c, 92, c) - d;
      if (q >= end) break;
      if (Memory.default.getU8(q + d) == c) {
        finish(T_STR, q + 1, start);
        return;
      }
      q = q + 2;
    }
    if (fin == 0) {
      needMore(start);
      return;
    }
    finish(T_STR, end, start);
    return;
  }

  if (c == 96) {
    scanTemplate(p + 1, T_TEMPLATE, T_TEMPLATE_HEAD, start);
    return;
  }

  if (c == 35) {
    var q = p + 1;
    while (q < end) {
      let x = Memory.default.getU8(q + d);
      if (isIdChar(x) || x >= 0x80) q = q + 1;
      else break;
    }
    finish(T_PRIVATE, q, start);
    return;
  }

  let c1 = at(p + 1);
  if (c == 46) {
    if (c1 == 46 && at(p + 2) == 46) {
      finish(T_ELLIPSIS, p + 3, start);
      return;
    }
    finish(T_DOT, p + 1, start);
    return;
  }
  if (c == 63 && c1 == 46 && !isDigit(at(p + 2))) { finish(T_QDOT, p + 2, start); return; }
  if (c == 61 && c1 == 62) { finish(T_ARROW, p + 2, start); return; }
  // Every other punctuator from PUNCT's entry for c (engine/keywords.wat): the byte alone, doubled, or with `=`,
  // then a further `=` where that forms a longer one. A byte with no entry (a stray control byte) is a
  // one-byte token. `>` is always a lone `>`; the expression parser widens it with rescanGreater.
  let e = PUNCT_ENT + Memory.default.getU8(PUNCT + c) * 5;
  var q = p + 1;
  var t = Memory.default.getU8(e);
  var x = 0;
  if (c1 == c && Memory.default.getU8(e + 3) != 0) {
    q = p + 2;
    t = Memory.default.getU8(e + 3);
    x = Memory.default.getU8(e + 4);
  } else if (c1 == 61 && Memory.default.getU8(e + 1) != 0) {
    q = p + 2;
    t = Memory.default.getU8(e + 1);
    x = Memory.default.getU8(e + 2);
  }
  if (x != 0 && at(q) == 61) {
    t = x;
    q = q + 1;
  }
  finish(t, q, start);
}

// Scans template text from q to a closing backtick (type whole) or `${` (type head).
function scanTemplate(q0: i32, whole: i32, head: i32, start: i32): void {
  let d = delta;
  let end = inEnd;
  var q = q0;
  while (q < end) {
    q = find3(q + d, end + d, 96, 36, 92) - d;
    if (q >= end) break;
    let x = Memory.default.getU8(q + d);
    if (x == 96) {
      finish(whole, q + 1, start);
      return;
    }
    if (x == 36) {
      if (q + 1 >= end) break;
      if (Memory.default.getU8(q + 1 + d) == 123) {
        finish(head, q + 2, start);
        return;
      }
      q = q + 1;
    } else {
      q = q + 2;
    }
  }
  if (fin == 0) {
    needMore(start);
    return;
  }
  finish(whole, end, start);
}

// The current token is `}` closing a template substitution: rescan it as the next template part.
export function rescanTemplate(): void {
  scanTemplate(ts + 1, T_TEMPLATE_TAIL, T_TEMPLATE_MID, tb);
}

// The current token is `/` or `/=` in operand position: rescan it as a regular expression.
export function rescanRegex(): void {
  let d = delta;
  let end = inEnd;
  var q = ts + 1;
  var inClass = false;
  var closed = false;
  while (q < end) {
    let x = Memory.default.getU8(q + d);
    if (x == 10 || x == 13) break;
    if (x == 92) {
      q = q + 2;
    } else if (x == 91) {
      inClass = true;
      q = q + 1;
    } else if (x == 93) {
      inClass = false;
      q = q + 1;
    } else if (x == 47 && !inClass) {
      q = q + 1;
      closed = true;
      break;
    } else {
      q = q + 1;
    }
  }
  if (!closed && fin == 0) {
    needMore(tb);
    return;
  }
  while (q < end) {
    let x = Memory.default.getU8(q + d);
    if (isIdChar(x)) q = q + 1;
    else break;
  }
  finish(T_REGEX, q, tb);
}

// The current token is `<<` where type arguments may start: split off a lone `<`.
export function rescanLess(): void {
  finish(T_LT, ts + 1, tb);
}

// The current token is `>` in operator position: widen it to `>=`, `>>`, `>>>`, `>>=` or `>>>=`.
export function rescanGreater(): void {
  var q = ts + 1;
  var t = T_GT;
  if (at(q) == 62) {
    q = q + 1;
    t = T_SHR;
    if (at(q) == 62) {
      q = q + 1;
      t = T_USHR;
    }
    if (at(q) == 61) {
      q = q + 1;
      if (t == T_SHR) t = T_SHR_EQ;
      else t = T_USHR_EQ;
    }
  } else if (at(q) == 61) {
    q = q + 1;
    t = T_GTE;
  }
  finish(t, q, tb);
}