diffdown.com
Diffdown is a real-time collaborative Markdown editor/previewer built on the AT Protocol
diffdown.com
1/* Hand-written tokenizers for JavaScript tokens that can't be
2 expressed by lezer's built-in tokenizer. */
3
4import {ExternalTokenizer, ContextTracker} from "@lezer/lr"
5import {insertSemi, noSemi, noSemiType, incdec, incdecPrefix, questionDot,
6 spaces, newline, BlockComment, LineComment,
7 JSXStartTag, Dialect_jsx} from "./parser.terms.js"
8
9const space = [9, 10, 11, 12, 13, 32, 133, 160, 5760, 8192, 8193, 8194, 8195, 8196, 8197, 8198, 8199, 8200,
10 8201, 8202, 8232, 8233, 8239, 8287, 12288]
11
12const braceR = 125, semicolon = 59, slash = 47, star = 42, plus = 43, minus = 45, lt = 60, comma = 44,
13 question = 63, dot = 46, bracketL = 91
14
15export const trackNewline = new ContextTracker({
16 start: false,
17 shift(context, term) {
18 return term == LineComment || term == BlockComment || term == spaces ? context : term == newline
19 },
20 strict: false
21})
22
23export const insertSemicolon = new ExternalTokenizer((input, stack) => {
24 let {next} = input
25 if (next == braceR || next == -1 || stack.context)
26 input.acceptToken(insertSemi)
27}, {contextual: true, fallback: true})
28
29export const noSemicolon = new ExternalTokenizer((input, stack) => {
30 let {next} = input, after
31 if (space.indexOf(next) > -1) return
32 if (next == slash && ((after = input.peek(1)) == slash || after == star)) return
33 if (next != braceR && next != semicolon && next != -1 && !stack.context)
34 input.acceptToken(noSemi)
35}, {contextual: true})
36
37export const noSemicolonType = new ExternalTokenizer((input, stack) => {
38 if (input.next == bracketL && !stack.context) input.acceptToken(noSemiType)
39}, {contextual: true})
40
41export const operatorToken = new ExternalTokenizer((input, stack) => {
42 let {next} = input
43 if (next == plus || next == minus) {
44 input.advance()
45 if (next == input.next) {
46 input.advance()
47 let mayPostfix = !stack.context && stack.canShift(incdec)
48 input.acceptToken(mayPostfix ? incdec : incdecPrefix)
49 }
50 } else if (next == question && input.peek(1) == dot) {
51 input.advance(); input.advance()
52 if (input.next < 48 || input.next > 57) // No digit after
53 input.acceptToken(questionDot)
54 }
55}, {contextual: true})
56
57function identifierChar(ch, start) {
58 return ch >= 65 && ch <= 90 || ch >= 97 && ch <= 122 || ch == 95 || ch >= 192 ||
59 !start && ch >= 48 && ch <= 57
60}
61
62export const jsx = new ExternalTokenizer((input, stack) => {
63 if (input.next != lt || !stack.dialectEnabled(Dialect_jsx)) return
64 input.advance()
65 if (input.next == slash) return
66 // Scan for an identifier followed by a comma or 'extends', don't
67 // treat this as a start tag if present.
68 let back = 0
69 while (space.indexOf(input.next) > -1) { input.advance(); back++ }
70 if (identifierChar(input.next, true)) {
71 input.advance()
72 back++
73 while (identifierChar(input.next, false)) { input.advance(); back++ }
74 while (space.indexOf(input.next) > -1) { input.advance(); back++ }
75 if (input.next == comma) return
76 for (let i = 0;; i++) {
77 if (i == 7) {
78 if (!identifierChar(input.next, true)) return
79 break
80 }
81 if (input.next != "extends".charCodeAt(i)) break
82 input.advance()
83 back++
84 }
85 }
86 input.acceptToken(JSXStartTag, -back)
87})