jcs.org
Serenity Operating System
1/*
2 * Copyright (c) 2020, Stephan Unverwerth <s.unverwerth@serenityos.org>
3 * Copyright (c) 2020-2021, Linus Groh <linusg@serenityos.org>
4 *
5 * SPDX-License-Identifier: BSD-2-Clause
6 */
7
8#include "Lexer.h"
9#include <AK/CharacterTypes.h>
10#include <AK/Debug.h>
11#include <AK/GenericLexer.h>
12#include <AK/HashMap.h>
13#include <AK/Utf8View.h>
14#include <LibUnicode/CharacterTypes.h>
15#include <stdio.h>
16
17namespace JS {
18
19HashMap<DeprecatedFlyString, TokenType> Lexer::s_keywords;
20HashMap<DeprecatedString, TokenType> Lexer::s_three_char_tokens;
21HashMap<DeprecatedString, TokenType> Lexer::s_two_char_tokens;
22HashMap<char, TokenType> Lexer::s_single_char_tokens;
23
24Lexer::Lexer(StringView source, StringView filename, size_t line_number, size_t line_column)
25 : m_source(source)
26 , m_current_token(TokenType::Eof, {}, {}, {}, filename, 0, 0, 0)
27 , m_filename(String::from_utf8(filename).release_value_but_fixme_should_propagate_errors())
28 , m_line_number(line_number)
29 , m_line_column(line_column)
30 , m_parsed_identifiers(adopt_ref(*new ParsedIdentifiers))
31{
32 if (s_keywords.is_empty()) {
33 s_keywords.set("async", TokenType::Async);
34 s_keywords.set("await", TokenType::Await);
35 s_keywords.set("break", TokenType::Break);
36 s_keywords.set("case", TokenType::Case);
37 s_keywords.set("catch", TokenType::Catch);
38 s_keywords.set("class", TokenType::Class);
39 s_keywords.set("const", TokenType::Const);
40 s_keywords.set("continue", TokenType::Continue);
41 s_keywords.set("debugger", TokenType::Debugger);
42 s_keywords.set("default", TokenType::Default);
43 s_keywords.set("delete", TokenType::Delete);
44 s_keywords.set("do", TokenType::Do);
45 s_keywords.set("else", TokenType::Else);
46 s_keywords.set("enum", TokenType::Enum);
47 s_keywords.set("export", TokenType::Export);
48 s_keywords.set("extends", TokenType::Extends);
49 s_keywords.set("false", TokenType::BoolLiteral);
50 s_keywords.set("finally", TokenType::Finally);
51 s_keywords.set("for", TokenType::For);
52 s_keywords.set("function", TokenType::Function);
53 s_keywords.set("if", TokenType::If);
54 s_keywords.set("import", TokenType::Import);
55 s_keywords.set("in", TokenType::In);
56 s_keywords.set("instanceof", TokenType::Instanceof);
57 s_keywords.set("let", TokenType::Let);
58 s_keywords.set("new", TokenType::New);
59 s_keywords.set("null", TokenType::NullLiteral);
60 s_keywords.set("return", TokenType::Return);
61 s_keywords.set("super", TokenType::Super);
62 s_keywords.set("switch", TokenType::Switch);
63 s_keywords.set("this", TokenType::This);
64 s_keywords.set("throw", TokenType::Throw);
65 s_keywords.set("true", TokenType::BoolLiteral);
66 s_keywords.set("try", TokenType::Try);
67 s_keywords.set("typeof", TokenType::Typeof);
68 s_keywords.set("var", TokenType::Var);
69 s_keywords.set("void", TokenType::Void);
70 s_keywords.set("while", TokenType::While);
71 s_keywords.set("with", TokenType::With);
72 s_keywords.set("yield", TokenType::Yield);
73 }
74
75 if (s_three_char_tokens.is_empty()) {
76 s_three_char_tokens.set("===", TokenType::EqualsEqualsEquals);
77 s_three_char_tokens.set("!==", TokenType::ExclamationMarkEqualsEquals);
78 s_three_char_tokens.set("**=", TokenType::DoubleAsteriskEquals);
79 s_three_char_tokens.set("<<=", TokenType::ShiftLeftEquals);
80 s_three_char_tokens.set(">>=", TokenType::ShiftRightEquals);
81 s_three_char_tokens.set("&&=", TokenType::DoubleAmpersandEquals);
82 s_three_char_tokens.set("||=", TokenType::DoublePipeEquals);
83 s_three_char_tokens.set("\?\?=", TokenType::DoubleQuestionMarkEquals);
84 s_three_char_tokens.set(">>>", TokenType::UnsignedShiftRight);
85 s_three_char_tokens.set("...", TokenType::TripleDot);
86 }
87
88 if (s_two_char_tokens.is_empty()) {
89 s_two_char_tokens.set("=>", TokenType::Arrow);
90 s_two_char_tokens.set("+=", TokenType::PlusEquals);
91 s_two_char_tokens.set("-=", TokenType::MinusEquals);
92 s_two_char_tokens.set("*=", TokenType::AsteriskEquals);
93 s_two_char_tokens.set("/=", TokenType::SlashEquals);
94 s_two_char_tokens.set("%=", TokenType::PercentEquals);
95 s_two_char_tokens.set("&=", TokenType::AmpersandEquals);
96 s_two_char_tokens.set("|=", TokenType::PipeEquals);
97 s_two_char_tokens.set("^=", TokenType::CaretEquals);
98 s_two_char_tokens.set("&&", TokenType::DoubleAmpersand);
99 s_two_char_tokens.set("||", TokenType::DoublePipe);
100 s_two_char_tokens.set("??", TokenType::DoubleQuestionMark);
101 s_two_char_tokens.set("**", TokenType::DoubleAsterisk);
102 s_two_char_tokens.set("==", TokenType::EqualsEquals);
103 s_two_char_tokens.set("<=", TokenType::LessThanEquals);
104 s_two_char_tokens.set(">=", TokenType::GreaterThanEquals);
105 s_two_char_tokens.set("!=", TokenType::ExclamationMarkEquals);
106 s_two_char_tokens.set("--", TokenType::MinusMinus);
107 s_two_char_tokens.set("++", TokenType::PlusPlus);
108 s_two_char_tokens.set("<<", TokenType::ShiftLeft);
109 s_two_char_tokens.set(">>", TokenType::ShiftRight);
110 s_two_char_tokens.set("?.", TokenType::QuestionMarkPeriod);
111 }
112
113 if (s_single_char_tokens.is_empty()) {
114 s_single_char_tokens.set('&', TokenType::Ampersand);
115 s_single_char_tokens.set('*', TokenType::Asterisk);
116 s_single_char_tokens.set('[', TokenType::BracketOpen);
117 s_single_char_tokens.set(']', TokenType::BracketClose);
118 s_single_char_tokens.set('^', TokenType::Caret);
119 s_single_char_tokens.set(':', TokenType::Colon);
120 s_single_char_tokens.set(',', TokenType::Comma);
121 s_single_char_tokens.set('{', TokenType::CurlyOpen);
122 s_single_char_tokens.set('}', TokenType::CurlyClose);
123 s_single_char_tokens.set('=', TokenType::Equals);
124 s_single_char_tokens.set('!', TokenType::ExclamationMark);
125 s_single_char_tokens.set('-', TokenType::Minus);
126 s_single_char_tokens.set('(', TokenType::ParenOpen);
127 s_single_char_tokens.set(')', TokenType::ParenClose);
128 s_single_char_tokens.set('%', TokenType::Percent);
129 s_single_char_tokens.set('.', TokenType::Period);
130 s_single_char_tokens.set('|', TokenType::Pipe);
131 s_single_char_tokens.set('+', TokenType::Plus);
132 s_single_char_tokens.set('?', TokenType::QuestionMark);
133 s_single_char_tokens.set(';', TokenType::Semicolon);
134 s_single_char_tokens.set('/', TokenType::Slash);
135 s_single_char_tokens.set('~', TokenType::Tilde);
136 s_single_char_tokens.set('<', TokenType::LessThan);
137 s_single_char_tokens.set('>', TokenType::GreaterThan);
138 }
139 consume();
140}
141
142void Lexer::consume()
143{
144 auto did_reach_eof = [this] {
145 if (m_position < m_source.length())
146 return false;
147 m_eof = true;
148 m_current_char = '\0';
149 m_position = m_source.length() + 1;
150 m_line_column++;
151 return true;
152 };
153
154 if (m_position > m_source.length())
155 return;
156
157 if (did_reach_eof())
158 return;
159
160 if (is_line_terminator()) {
161 if constexpr (LEXER_DEBUG) {
162 DeprecatedString type;
163 if (m_current_char == '\n')
164 type = "LINE FEED";
165 else if (m_current_char == '\r')
166 type = "CARRIAGE RETURN";
167 else if (m_source[m_position + 1] == (char)0xa8)
168 type = "LINE SEPARATOR";
169 else
170 type = "PARAGRAPH SEPARATOR";
171 dbgln("Found a line terminator: {}", type);
172 }
173 // This is a three-char line terminator, we need to increase m_position some more.
174 // We might reach EOF and need to check again.
175 if (m_current_char != '\n' && m_current_char != '\r') {
176 m_position += 2;
177 if (did_reach_eof())
178 return;
179 }
180
181 // If the previous character is \r and the current one \n we already updated line number
182 // and column - don't do it again. From https://tc39.es/ecma262/#sec-line-terminators:
183 // The sequence <CR><LF> is commonly used as a line terminator.
184 // It should be considered a single SourceCharacter for the purpose of reporting line numbers.
185 auto second_char_of_crlf = m_position > 1 && m_source[m_position - 2] == '\r' && m_current_char == '\n';
186
187 if (!second_char_of_crlf) {
188 m_line_number++;
189 m_line_column = 1;
190 dbgln_if(LEXER_DEBUG, "Incremented line number, now at: line {}, column 1", m_line_number);
191 } else {
192 dbgln_if(LEXER_DEBUG, "Previous was CR, this is LF - not incrementing line number again.");
193 }
194 } else if (is_unicode_character()) {
195 size_t char_size = 1;
196 if ((m_current_char & 64) == 0) {
197 m_hit_invalid_unicode = m_position;
198 } else if ((m_current_char & 32) == 0) {
199 char_size = 2;
200 } else if ((m_current_char & 16) == 0) {
201 char_size = 3;
202 } else if ((m_current_char & 8) == 0) {
203 char_size = 4;
204 }
205
206 VERIFY(char_size >= 1);
207 --char_size;
208
209 for (size_t i = m_position; i < m_position + char_size; i++) {
210 if (i >= m_source.length() || (m_source[i] & 0b11000000) != 0b10000000) {
211 m_hit_invalid_unicode = m_position;
212 break;
213 }
214 }
215
216 if (m_hit_invalid_unicode.has_value())
217 m_position = m_source.length();
218 else
219 m_position += char_size;
220
221 if (did_reach_eof())
222 return;
223
224 m_line_column++;
225 } else {
226 m_line_column++;
227 }
228
229 m_current_char = m_source[m_position++];
230}
231
232bool Lexer::consume_decimal_number()
233{
234 if (!is_ascii_digit(m_current_char))
235 return false;
236
237 while (is_ascii_digit(m_current_char) || match_numeric_literal_separator_followed_by(is_ascii_digit)) {
238 consume();
239 }
240 return true;
241}
242
243bool Lexer::consume_exponent()
244{
245 consume();
246 if (m_current_char == '-' || m_current_char == '+')
247 consume();
248
249 if (!is_ascii_digit(m_current_char))
250 return false;
251
252 return consume_decimal_number();
253}
254
255static constexpr bool is_octal_digit(char ch)
256{
257 return ch >= '0' && ch <= '7';
258}
259
260bool Lexer::consume_octal_number()
261{
262 consume();
263 if (!is_octal_digit(m_current_char))
264 return false;
265
266 while (is_octal_digit(m_current_char) || match_numeric_literal_separator_followed_by(is_octal_digit))
267 consume();
268
269 return true;
270}
271
272bool Lexer::consume_hexadecimal_number()
273{
274 consume();
275 if (!is_ascii_hex_digit(m_current_char))
276 return false;
277
278 while (is_ascii_hex_digit(m_current_char) || match_numeric_literal_separator_followed_by(is_ascii_hex_digit))
279 consume();
280
281 return true;
282}
283
284static constexpr bool is_binary_digit(char ch)
285{
286 return ch == '0' || ch == '1';
287}
288
289bool Lexer::consume_binary_number()
290{
291 consume();
292 if (!is_binary_digit(m_current_char))
293 return false;
294
295 while (is_binary_digit(m_current_char) || match_numeric_literal_separator_followed_by(is_binary_digit))
296 consume();
297
298 return true;
299}
300
301template<typename Callback>
302bool Lexer::match_numeric_literal_separator_followed_by(Callback callback) const
303{
304 if (m_position >= m_source.length())
305 return false;
306 return m_current_char == '_'
307 && callback(m_source[m_position]);
308}
309
310bool Lexer::match(char a, char b) const
311{
312 if (m_position >= m_source.length())
313 return false;
314
315 return m_current_char == a
316 && m_source[m_position] == b;
317}
318
319bool Lexer::match(char a, char b, char c) const
320{
321 if (m_position + 1 >= m_source.length())
322 return false;
323
324 return m_current_char == a
325 && m_source[m_position] == b
326 && m_source[m_position + 1] == c;
327}
328
329bool Lexer::match(char a, char b, char c, char d) const
330{
331 if (m_position + 2 >= m_source.length())
332 return false;
333
334 return m_current_char == a
335 && m_source[m_position] == b
336 && m_source[m_position + 1] == c
337 && m_source[m_position + 2] == d;
338}
339
340bool Lexer::is_eof() const
341{
342 return m_eof;
343}
344
345ALWAYS_INLINE bool Lexer::is_line_terminator() const
346{
347 if (m_current_char == '\n' || m_current_char == '\r')
348 return true;
349 if (!is_unicode_character())
350 return false;
351
352 auto code_point = current_code_point();
353 return code_point == LINE_SEPARATOR || code_point == PARAGRAPH_SEPARATOR;
354}
355
356ALWAYS_INLINE bool Lexer::is_unicode_character() const
357{
358 return (m_current_char & 128) != 0;
359}
360
361ALWAYS_INLINE u32 Lexer::current_code_point() const
362{
363 static constexpr const u32 REPLACEMENT_CHARACTER = 0xFFFD;
364 if (m_position == 0)
365 return REPLACEMENT_CHARACTER;
366 auto substring = m_source.substring_view(m_position - 1);
367 if (substring.is_empty())
368 return REPLACEMENT_CHARACTER;
369 if (is_ascii(substring[0]))
370 return substring[0];
371 Utf8View utf_8_view { substring };
372 return *utf_8_view.begin();
373}
374
375bool Lexer::is_whitespace() const
376{
377 if (is_ascii_space(m_current_char))
378 return true;
379 if (!is_unicode_character())
380 return false;
381 auto code_point = current_code_point();
382 if (code_point == NO_BREAK_SPACE || code_point == ZERO_WIDTH_NO_BREAK_SPACE)
383 return true;
384
385 static auto space_separator_category = Unicode::general_category_from_string("Space_Separator"sv);
386 if (space_separator_category.has_value())
387 return Unicode::code_point_has_general_category(code_point, *space_separator_category);
388 return false;
389}
390
391// UnicodeEscapeSequence :: https://tc39.es/ecma262/#prod-UnicodeEscapeSequence
392// u Hex4Digits
393// u{ CodePoint }
394Optional<u32> Lexer::is_identifier_unicode_escape(size_t& identifier_length) const
395{
396 GenericLexer lexer(source().substring_view(m_position - 1));
397
398 if (auto code_point_or_error = lexer.consume_escaped_code_point(false); !code_point_or_error.is_error()) {
399 identifier_length = lexer.tell();
400 return code_point_or_error.value();
401 }
402
403 return {};
404}
405
406// IdentifierStart :: https://tc39.es/ecma262/#prod-IdentifierStart
407// UnicodeIDStart
408// $
409// _
410// \ UnicodeEscapeSequence
411Optional<u32> Lexer::is_identifier_start(size_t& identifier_length) const
412{
413 u32 code_point = current_code_point();
414 identifier_length = 1;
415
416 if (code_point == '\\') {
417 if (auto maybe_code_point = is_identifier_unicode_escape(identifier_length); maybe_code_point.has_value())
418 code_point = *maybe_code_point;
419 else
420 return {};
421 }
422
423 if (is_ascii_alpha(code_point) || code_point == '_' || code_point == '$')
424 return code_point;
425
426 // Optimization: the first codepoint with the ID_Start property after A-Za-z is outside the
427 // ASCII range (0x00AA), so we can skip code_point_has_property() for any ASCII characters.
428 if (is_ascii(code_point))
429 return {};
430
431 static auto id_start_category = Unicode::property_from_string("ID_Start"sv);
432 if (id_start_category.has_value() && Unicode::code_point_has_property(code_point, *id_start_category))
433 return code_point;
434
435 return {};
436}
437
438// IdentifierPart :: https://tc39.es/ecma262/#prod-IdentifierPart
439// UnicodeIDContinue
440// $
441// \ UnicodeEscapeSequence
442// <ZWNJ>
443// <ZWJ>
444Optional<u32> Lexer::is_identifier_middle(size_t& identifier_length) const
445{
446 u32 code_point = current_code_point();
447 identifier_length = 1;
448
449 if (code_point == '\\') {
450 if (auto maybe_code_point = is_identifier_unicode_escape(identifier_length); maybe_code_point.has_value())
451 code_point = *maybe_code_point;
452 else
453 return {};
454 }
455
456 if (is_ascii_alphanumeric(code_point) || (code_point == '$') || (code_point == ZERO_WIDTH_NON_JOINER) || (code_point == ZERO_WIDTH_JOINER))
457 return code_point;
458
459 // Optimization: the first codepoint with the ID_Continue property after A-Za-z0-9_ is outside the
460 // ASCII range (0x00AA), so we can skip code_point_has_property() for any ASCII characters.
461 if (code_point == '_')
462 return code_point;
463 if (is_ascii(code_point))
464 return {};
465
466 static auto id_continue_category = Unicode::property_from_string("ID_Continue"sv);
467 if (id_continue_category.has_value() && Unicode::code_point_has_property(code_point, *id_continue_category))
468 return code_point;
469
470 return {};
471}
472
473bool Lexer::is_line_comment_start(bool line_has_token_yet) const
474{
475 return match('/', '/')
476 || (m_allow_html_comments && match('<', '!', '-', '-'))
477 // "-->" is considered a line comment start if the current line is only whitespace and/or
478 // other block comment(s); or in other words: the current line does not have a token or
479 // ongoing line comment yet
480 || (m_allow_html_comments && !line_has_token_yet && match('-', '-', '>'))
481 // https://tc39.es/proposal-hashbang/out.html#sec-updated-syntax
482 || (match('#', '!') && m_position == 1);
483}
484
485bool Lexer::is_block_comment_start() const
486{
487 return match('/', '*');
488}
489
490bool Lexer::is_block_comment_end() const
491{
492 return match('*', '/');
493}
494
495bool Lexer::is_numeric_literal_start() const
496{
497 return is_ascii_digit(m_current_char) || (m_current_char == '.' && m_position < m_source.length() && is_ascii_digit(m_source[m_position]));
498}
499
500bool Lexer::slash_means_division() const
501{
502 auto type = m_current_token.type();
503 return type == TokenType::BigIntLiteral
504 || type == TokenType::BoolLiteral
505 || type == TokenType::BracketClose
506 || type == TokenType::CurlyClose
507 || type == TokenType::Identifier
508 || type == TokenType::In
509 || type == TokenType::Instanceof
510 || type == TokenType::MinusMinus
511 || type == TokenType::NullLiteral
512 || type == TokenType::NumericLiteral
513 || type == TokenType::ParenClose
514 || type == TokenType::PlusPlus
515 || type == TokenType::PrivateIdentifier
516 || type == TokenType::RegexLiteral
517 || type == TokenType::StringLiteral
518 || type == TokenType::TemplateLiteralEnd
519 || type == TokenType::This;
520}
521
522Token Lexer::next()
523{
524 size_t trivia_start = m_position;
525 auto in_template = !m_template_states.is_empty();
526 bool line_has_token_yet = m_line_column > 1;
527 bool unterminated_comment = false;
528
529 if (!in_template || m_template_states.last().in_expr) {
530 // consume whitespace and comments
531 while (true) {
532 if (is_line_terminator()) {
533 line_has_token_yet = false;
534 do {
535 consume();
536 } while (is_line_terminator());
537 } else if (is_whitespace()) {
538 do {
539 consume();
540 } while (is_whitespace());
541 } else if (is_line_comment_start(line_has_token_yet)) {
542 consume();
543 do {
544 consume();
545 } while (!is_eof() && !is_line_terminator());
546 } else if (is_block_comment_start()) {
547 size_t start_line_number = m_line_number;
548 consume();
549 do {
550 consume();
551 } while (!is_eof() && !is_block_comment_end());
552 if (is_eof())
553 unterminated_comment = true;
554 consume(); // consume *
555 if (is_eof())
556 unterminated_comment = true;
557 consume(); // consume /
558
559 if (start_line_number != m_line_number)
560 line_has_token_yet = false;
561 } else {
562 break;
563 }
564 }
565 }
566
567 size_t value_start = m_position;
568 size_t value_start_line_number = m_line_number;
569 size_t value_start_column_number = m_line_column;
570 auto token_type = TokenType::Invalid;
571 auto did_consume_whitespace_or_comments = trivia_start != value_start;
572 // This is being used to communicate info about invalid tokens to the parser, which then
573 // can turn that into more specific error messages - instead of us having to make up a
574 // bunch of Invalid* tokens (bad numeric literals, unterminated comments etc.)
575 DeprecatedString token_message;
576
577 Optional<DeprecatedFlyString> identifier;
578 size_t identifier_length = 0;
579
580 if (m_current_token.type() == TokenType::RegexLiteral && !is_eof() && is_ascii_alpha(m_current_char) && !did_consume_whitespace_or_comments) {
581 token_type = TokenType::RegexFlags;
582 while (!is_eof() && is_ascii_alpha(m_current_char))
583 consume();
584 } else if (m_current_char == '`') {
585 consume();
586
587 if (!in_template) {
588 token_type = TokenType::TemplateLiteralStart;
589 m_template_states.append({ false, 0 });
590 } else {
591 if (m_template_states.last().in_expr) {
592 m_template_states.append({ false, 0 });
593 token_type = TokenType::TemplateLiteralStart;
594 } else {
595 m_template_states.take_last();
596 token_type = TokenType::TemplateLiteralEnd;
597 }
598 }
599 } else if (in_template && m_template_states.last().in_expr && m_template_states.last().open_bracket_count == 0 && m_current_char == '}') {
600 consume();
601 token_type = TokenType::TemplateLiteralExprEnd;
602 m_template_states.last().in_expr = false;
603 } else if (in_template && !m_template_states.last().in_expr) {
604 if (is_eof()) {
605 token_type = TokenType::UnterminatedTemplateLiteral;
606 m_template_states.take_last();
607 } else if (match('$', '{')) {
608 token_type = TokenType::TemplateLiteralExprStart;
609 consume();
610 consume();
611 m_template_states.last().in_expr = true;
612 } else {
613 // TemplateCharacter ::
614 // $ [lookahead ≠ {]
615 // \ TemplateEscapeSequence
616 // \ NotEscapeSequence
617 // LineContinuation
618 // LineTerminatorSequence
619 // SourceCharacter but not one of ` or \ or $ or LineTerminator
620 while (!match('$', '{') && m_current_char != '`' && !is_eof()) {
621 if (match('\\', '$') || match('\\', '`') || match('\\', '\\'))
622 consume();
623 consume();
624 }
625 if (is_eof() && !m_template_states.is_empty())
626 token_type = TokenType::UnterminatedTemplateLiteral;
627 else
628 token_type = TokenType::TemplateLiteralString;
629 }
630 } else if (m_current_char == '#') {
631 // Note: This has some duplicated code with the identifier lexing below
632 consume();
633 auto code_point = is_identifier_start(identifier_length);
634 if (code_point.has_value()) {
635 StringBuilder builder;
636 builder.append_code_point('#');
637 do {
638 builder.append_code_point(*code_point);
639 for (size_t i = 0; i < identifier_length; ++i)
640 consume();
641
642 code_point = is_identifier_middle(identifier_length);
643 } while (code_point.has_value());
644
645 identifier = builder.string_view();
646 token_type = TokenType::PrivateIdentifier;
647
648 m_parsed_identifiers->identifiers.set(*identifier);
649 } else {
650 token_type = TokenType::Invalid;
651 token_message = "Start of private name '#' but not followed by valid identifier";
652 }
653 } else if (auto code_point = is_identifier_start(identifier_length); code_point.has_value()) {
654 bool has_escaped_character = false;
655 // identifier or keyword
656 StringBuilder builder;
657 do {
658 builder.append_code_point(*code_point);
659 for (size_t i = 0; i < identifier_length; ++i)
660 consume();
661
662 has_escaped_character |= identifier_length > 1;
663
664 code_point = is_identifier_middle(identifier_length);
665 } while (code_point.has_value());
666
667 identifier = builder.string_view();
668 m_parsed_identifiers->identifiers.set(*identifier);
669
670 auto it = s_keywords.find(identifier->hash(), [&](auto& entry) { return entry.key == identifier; });
671 if (it == s_keywords.end())
672 token_type = TokenType::Identifier;
673 else
674 token_type = has_escaped_character ? TokenType::EscapedKeyword : it->value;
675 } else if (is_numeric_literal_start()) {
676 token_type = TokenType::NumericLiteral;
677 bool is_invalid_numeric_literal = false;
678 if (m_current_char == '0') {
679 consume();
680 if (m_current_char == '.') {
681 // decimal
682 consume();
683 while (is_ascii_digit(m_current_char))
684 consume();
685 if (m_current_char == 'e' || m_current_char == 'E')
686 is_invalid_numeric_literal = !consume_exponent();
687 } else if (m_current_char == 'e' || m_current_char == 'E') {
688 is_invalid_numeric_literal = !consume_exponent();
689 } else if (m_current_char == 'o' || m_current_char == 'O') {
690 // octal
691 is_invalid_numeric_literal = !consume_octal_number();
692 if (m_current_char == 'n') {
693 consume();
694 token_type = TokenType::BigIntLiteral;
695 }
696 } else if (m_current_char == 'b' || m_current_char == 'B') {
697 // binary
698 is_invalid_numeric_literal = !consume_binary_number();
699 if (m_current_char == 'n') {
700 consume();
701 token_type = TokenType::BigIntLiteral;
702 }
703 } else if (m_current_char == 'x' || m_current_char == 'X') {
704 // hexadecimal
705 is_invalid_numeric_literal = !consume_hexadecimal_number();
706 if (m_current_char == 'n') {
707 consume();
708 token_type = TokenType::BigIntLiteral;
709 }
710 } else if (m_current_char == 'n') {
711 consume();
712 token_type = TokenType::BigIntLiteral;
713 } else if (is_ascii_digit(m_current_char)) {
714 // octal without '0o' prefix. Forbidden in 'strict mode'
715 do {
716 consume();
717 } while (is_ascii_digit(m_current_char));
718 }
719 } else {
720 // 1...9 or period
721 while (is_ascii_digit(m_current_char) || match_numeric_literal_separator_followed_by(is_ascii_digit))
722 consume();
723 if (m_current_char == 'n') {
724 consume();
725 token_type = TokenType::BigIntLiteral;
726 } else {
727 if (m_current_char == '.') {
728 consume();
729 if (m_current_char == '_')
730 is_invalid_numeric_literal = true;
731
732 while (is_ascii_digit(m_current_char) || match_numeric_literal_separator_followed_by(is_ascii_digit)) {
733 consume();
734 }
735 }
736 if (m_current_char == 'e' || m_current_char == 'E')
737 is_invalid_numeric_literal = is_invalid_numeric_literal || !consume_exponent();
738 }
739 }
740 if (is_invalid_numeric_literal) {
741 token_type = TokenType::Invalid;
742 token_message = "Invalid numeric literal";
743 }
744 } else if (m_current_char == '"' || m_current_char == '\'') {
745 char stop_char = m_current_char;
746 consume();
747 // Note: LS/PS line terminators are allowed in string literals.
748 while (m_current_char != stop_char && m_current_char != '\r' && m_current_char != '\n' && !is_eof()) {
749 if (m_current_char == '\\') {
750 consume();
751 if (m_current_char == '\r' && m_position < m_source.length() && m_source[m_position] == '\n') {
752 consume();
753 }
754 }
755 consume();
756 }
757 if (m_current_char != stop_char) {
758 token_type = TokenType::UnterminatedStringLiteral;
759 } else {
760 consume();
761 token_type = TokenType::StringLiteral;
762 }
763 } else if (m_current_char == '/' && !slash_means_division()) {
764 consume();
765 token_type = consume_regex_literal();
766 } else if (m_eof) {
767 if (unterminated_comment) {
768 token_type = TokenType::Invalid;
769 token_message = "Unterminated multi-line comment";
770 } else {
771 token_type = TokenType::Eof;
772 }
773 } else {
774 // There is only one four-char operator: >>>=
775 bool found_four_char_token = false;
776 if (match('>', '>', '>', '=')) {
777 found_four_char_token = true;
778 consume();
779 consume();
780 consume();
781 consume();
782 token_type = TokenType::UnsignedShiftRightEquals;
783 }
784
785 bool found_three_char_token = false;
786 if (!found_four_char_token && m_position + 1 < m_source.length()) {
787 auto three_chars_view = m_source.substring_view(m_position - 1, 3);
788 auto it = s_three_char_tokens.find(three_chars_view.hash(), [&](auto& entry) { return entry.key == three_chars_view; });
789 if (it != s_three_char_tokens.end()) {
790 found_three_char_token = true;
791 consume();
792 consume();
793 consume();
794 token_type = it->value;
795 }
796 }
797
798 bool found_two_char_token = false;
799 if (!found_four_char_token && !found_three_char_token && m_position < m_source.length()) {
800 auto two_chars_view = m_source.substring_view(m_position - 1, 2);
801 auto it = s_two_char_tokens.find(two_chars_view.hash(), [&](auto& entry) { return entry.key == two_chars_view; });
802 if (it != s_two_char_tokens.end()) {
803 // OptionalChainingPunctuator :: ?. [lookahead ∉ DecimalDigit]
804 if (!(it->value == TokenType::QuestionMarkPeriod && m_position + 1 < m_source.length() && is_ascii_digit(m_source[m_position + 1]))) {
805 found_two_char_token = true;
806 consume();
807 consume();
808 token_type = it->value;
809 }
810 }
811 }
812
813 bool found_one_char_token = false;
814 if (!found_four_char_token && !found_three_char_token && !found_two_char_token) {
815 auto it = s_single_char_tokens.find(m_current_char);
816 if (it != s_single_char_tokens.end()) {
817 found_one_char_token = true;
818 consume();
819 token_type = it->value;
820 }
821 }
822
823 if (!found_four_char_token && !found_three_char_token && !found_two_char_token && !found_one_char_token) {
824 consume();
825 token_type = TokenType::Invalid;
826 }
827 }
828
829 if (!m_template_states.is_empty() && m_template_states.last().in_expr) {
830 if (token_type == TokenType::CurlyOpen) {
831 m_template_states.last().open_bracket_count++;
832 } else if (token_type == TokenType::CurlyClose) {
833 m_template_states.last().open_bracket_count--;
834 }
835 }
836
837 if (m_hit_invalid_unicode.has_value()) {
838 value_start = m_hit_invalid_unicode.value() - 1;
839 m_current_token = Token(TokenType::Invalid, "Invalid unicode codepoint in source"_string.release_value_but_fixme_should_propagate_errors(),
840 ""sv, // Since the invalid unicode can occur anywhere in the current token the trivia is not correct
841 m_source.substring_view(value_start + 1, min(4u, m_source.length() - value_start - 2)),
842 m_filename,
843 m_line_number,
844 m_line_column - 1,
845 value_start + 1);
846 m_hit_invalid_unicode.clear();
847 // Do not produce any further tokens.
848 VERIFY(is_eof());
849 } else {
850 m_current_token = Token(
851 token_type,
852 String::from_deprecated_string(token_message).release_value_but_fixme_should_propagate_errors(),
853 m_source.substring_view(trivia_start - 1, value_start - trivia_start),
854 m_source.substring_view(value_start - 1, m_position - value_start),
855 m_filename,
856 value_start_line_number,
857 value_start_column_number,
858 value_start - 1);
859 }
860
861 if (identifier.has_value())
862 m_current_token.set_identifier_value(identifier.release_value());
863
864 if constexpr (LEXER_DEBUG) {
865 dbgln("------------------------------");
866 dbgln("Token: {}", m_current_token.name());
867 dbgln("Trivia: _{}_", m_current_token.trivia());
868 dbgln("Value: _{}_", m_current_token.value());
869 dbgln("Line: {}, Column: {}", m_current_token.line_number(), m_current_token.line_column());
870 dbgln("------------------------------");
871 }
872
873 return m_current_token;
874}
875
876Token Lexer::force_slash_as_regex()
877{
878 VERIFY(m_current_token.type() == TokenType::Slash || m_current_token.type() == TokenType::SlashEquals);
879
880 bool has_equals = m_current_token.type() == TokenType::SlashEquals;
881
882 VERIFY(m_position > 0);
883 size_t value_start = m_position - 1;
884
885 if (has_equals) {
886 VERIFY(m_source[value_start - 1] == '=');
887 --value_start;
888 --m_position;
889 m_current_char = '=';
890 }
891
892 TokenType token_type = consume_regex_literal();
893
894 m_current_token = Token(
895 token_type,
896 String {},
897 m_current_token.trivia(),
898 m_source.substring_view(value_start - 1, m_position - value_start),
899 m_filename,
900 m_current_token.line_number(),
901 m_current_token.line_column(),
902 value_start - 1);
903
904 if constexpr (LEXER_DEBUG) {
905 dbgln("------------------------------");
906 dbgln("Token: {}", m_current_token.name());
907 dbgln("Trivia: _{}_", m_current_token.trivia());
908 dbgln("Value: _{}_", m_current_token.value());
909 dbgln("Line: {}, Column: {}", m_current_token.line_number(), m_current_token.line_column());
910 dbgln("------------------------------");
911 }
912
913 return m_current_token;
914}
915
916TokenType Lexer::consume_regex_literal()
917{
918 while (!is_eof()) {
919 if (is_line_terminator() || (!m_regex_is_in_character_class && m_current_char == '/')) {
920 break;
921 } else if (m_current_char == '[') {
922 m_regex_is_in_character_class = true;
923 } else if (m_current_char == ']') {
924 m_regex_is_in_character_class = false;
925 } else if (!m_regex_is_in_character_class && m_current_char == '/') {
926 break;
927 }
928
929 if (match('\\', '/') || match('\\', '[') || match('\\', '\\') || (m_regex_is_in_character_class && match('\\', ']')))
930 consume();
931 consume();
932 }
933
934 if (m_current_char == '/') {
935 consume();
936 return TokenType::RegexLiteral;
937 }
938
939 return TokenType::UnterminatedRegexLiteral;
940}
941
942}