we (web engine): Experimental web browser project to understand the limits of Claude

Merge js-lexer: fix line continuation escape in string and template literals

+38 -23
+38 -23
crates/js/src/lexer.rs
··· 673 673 } 674 674 Some(b'\\') => { 675 675 self.advance(); // backslash 676 - let ch = self.scan_escape_sequence()?; 677 - value.push(ch); 676 + if let Some(ch) = self.scan_escape_sequence()? { 677 + value.push(ch); 678 + } 678 679 } 679 680 Some(_) => { 680 681 let ch = self.advance_char(); ··· 686 687 Ok(TokenKind::String(value)) 687 688 } 688 689 689 - fn scan_escape_sequence(&mut self) -> Result<char, LexError> { 690 + /// Scan an escape sequence after the backslash has been consumed. 691 + /// Returns `None` for line continuations (`\<newline>`), which produce no character. 692 + fn scan_escape_sequence(&mut self) -> Result<Option<char>, LexError> { 690 693 let pos = self.current_pos(); 691 694 match self.advance() { 692 - Some(b'n') => Ok('\n'), 693 - Some(b'r') => Ok('\r'), 694 - Some(b't') => Ok('\t'), 695 - Some(b'b') => Ok('\u{0008}'), 696 - Some(b'f') => Ok('\u{000C}'), 697 - Some(b'v') => Ok('\u{000B}'), 698 - Some(b'0') if !matches!(self.peek(), Some(b'0'..=b'9')) => Ok('\0'), 699 - Some(b'\\') => Ok('\\'), 700 - Some(b'\'') => Ok('\''), 701 - Some(b'"') => Ok('"'), 702 - Some(b'`') => Ok('`'), 703 - Some(b'\n') => Ok('\n'), 695 + Some(b'n') => Ok(Some('\n')), 696 + Some(b'r') => Ok(Some('\r')), 697 + Some(b't') => Ok(Some('\t')), 698 + Some(b'b') => Ok(Some('\u{0008}')), 699 + Some(b'f') => Ok(Some('\u{000C}')), 700 + Some(b'v') => Ok(Some('\u{000B}')), 701 + Some(b'0') if !matches!(self.peek(), Some(b'0'..=b'9')) => Ok(Some('\0')), 702 + Some(b'\\') => Ok(Some('\\')), 703 + Some(b'\'') => Ok(Some('\'')), 704 + Some(b'"') => Ok(Some('"')), 705 + Some(b'`') => Ok(Some('`')), 706 + // Line continuation: \<newline> produces no character 707 + Some(b'\n') => Ok(None), 704 708 Some(b'\r') => { 705 709 self.advance_if(b'\n'); 706 - Ok('\n') 710 + Ok(None) 707 711 } 708 712 Some(b'x') => { 709 713 let hi = self.advance().and_then(hex_digit_val).ok_or(LexError { ··· 715 719 pos, 716 720 })?; 717 721 let code = (hi << 4) | lo; 718 - Ok(code as char) 722 + Ok(Some(code as char)) 719 723 } 720 - Some(b'u') => self.scan_unicode_escape(pos), 724 + Some(b'u') => self.scan_unicode_escape(pos).map(Some), 721 725 Some(b) => { 722 726 // identity escape 723 - Ok(b as char) 727 + Ok(Some(b as char)) 724 728 } 725 729 None => Err(LexError { 726 730 message: "unexpected end of input in escape sequence".into(), ··· 839 843 } 840 844 Some(b'\\') => { 841 845 self.advance(); 842 - let ch = self.scan_escape_sequence()?; 843 - value.push(ch); 846 + if let Some(ch) = self.scan_escape_sequence()? { 847 + value.push(ch); 848 + } 844 849 } 845 850 Some(_) => { 846 851 let ch = self.advance_char(); ··· 887 892 } 888 893 Some(b'\\') => { 889 894 self.advance(); 890 - let ch = self.scan_escape_sequence()?; 891 - value.push(ch); 895 + if let Some(ch) = self.scan_escape_sequence()? { 896 + value.push(ch); 897 + } 892 898 } 893 899 Some(_) => { 894 900 let ch = self.advance_char(); ··· 1535 1541 assert_eq!( 1536 1542 kind(r#""\u{1F600}""#), 1537 1543 TokenKind::String("\u{1F600}".into()) 1544 + ); 1545 + } 1546 + 1547 + #[test] 1548 + fn test_string_line_continuation() { 1549 + // \<newline> is a line continuation producing no character 1550 + assert_eq!( 1551 + kind("\"line1\\\nline2\""), 1552 + TokenKind::String("line1line2".into()) 1538 1553 ); 1539 1554 } 1540 1555