diffdown.com
Diffdown is a real-time collaborative Markdown editor/previewer built on the AT Protocol
diffdown.com
1/* Hand-written tokenizers for CSS tokens that can't be
2 expressed by Lezer's built-in tokenizer. */
3
4import {ExternalTokenizer} from "@lezer/lr"
5import {
6 identifier, callee, VariableName,
7 queryIdentifier, queryVariableName, QueryCallee,
8 descendantOp, Unit
9} from "./parser.terms.js"
10
11const space = [9, 10, 11, 12, 13, 32, 133, 160, 5760, 8192, 8193, 8194, 8195, 8196, 8197,
12 8198, 8199, 8200, 8201, 8202, 8232, 8233, 8239, 8287, 12288]
13const colon = 58, parenL = 40, underscore = 95, bracketL = 91, dash = 45, period = 46,
14 hash = 35, percent = 37, ampersand = 38, backslash = 92, newline = 10, asterisk = 42
15
16function isAlpha(ch) { return ch >= 65 && ch <= 90 || ch >= 97 && ch <= 122 || ch >= 161 }
17
18function isDigit(ch) { return ch >= 48 && ch <= 57 }
19
20function isHex(ch) { return isDigit(ch) || ch >= 97 && ch <= 102 || ch >= 65 && ch <= 70 }
21
22const identifierTokens = (id, varName, callee) => (input, stack) => {
23 for (let inside = false, dashes = 0, i = 0;; i++) {
24 let {next} = input
25 if (isAlpha(next) || next == dash || next == underscore || (inside && isDigit(next))) {
26 if (!inside && (next != dash || i > 0)) inside = true
27 if (dashes === i && next == dash) dashes++
28 input.advance()
29 } else if (next == backslash && input.peek(1) != newline) {
30 input.advance()
31 if (isHex(input.next)) {
32 do { input.advance() } while (isHex(input.next))
33 if (input.next == 32) input.advance()
34 } else if (input.next > -1) {
35 input.advance()
36 }
37 inside = true
38 } else {
39 if (inside) input.acceptToken(
40 dashes == 2 && stack.canShift(VariableName) ? varName : next == parenL ? callee : id
41 )
42 break
43 }
44 }
45}
46
47export const identifiers = new ExternalTokenizer(
48 identifierTokens(identifier, VariableName, callee)
49)
50export const queryIdentifiers = new ExternalTokenizer(
51 identifierTokens(queryIdentifier, queryVariableName, QueryCallee)
52)
53
54export const descendant = new ExternalTokenizer(input => {
55 if (space.includes(input.peek(-1))) {
56 let {next} = input
57 if (isAlpha(next) || next == underscore || next == hash || next == period ||
58 next == asterisk || next == bracketL || next == colon && isAlpha(input.peek(1)) ||
59 next == dash || next == ampersand)
60 input.acceptToken(descendantOp)
61 }
62})
63
64export const unitToken = new ExternalTokenizer(input => {
65 if (!space.includes(input.peek(-1))) {
66 let {next} = input
67 if (next == percent) { input.advance(); input.acceptToken(Unit) }
68 if (isAlpha(next)) {
69 do { input.advance() } while (isAlpha(input.next) || isDigit(input.next))
70 input.acceptToken(Unit)
71 }
72 }
73})