An Erlang lexer and syntax highlighter in Gleam
2
fork

Configure Feed

Select the types of activity you want to include in your feed.

Add support for triple-quoted strings with more than 3 quotes

+84 -7
+39 -4
src/pearl.gleam
··· 722 722 lexer: Lexer, 723 723 sigil: Option(String), 724 724 ) -> #(Lexer, Token) { 725 + let #(lexer, extra_quotes) = count_extra_quotes(lexer, 0) 726 + 725 727 let #(lexer, beginning_whitespace) = case 726 728 splitter.split(lexer.splitters.until_end_of_line, lexer.source) 727 729 { ··· 734 736 } 735 737 736 738 let #(lexer, lines, end_indentation) = 737 - lex_triple_quoted_string_contents(lexer, [], "") 739 + lex_triple_quoted_string_contents(lexer, [], "", extra_quotes) 738 740 739 741 case strip_line_prefixes(lines, end_indentation, []) { 740 742 Error(line) -> { ··· 758 760 lexer, 759 761 token.TripleQuotedString( 760 762 sigil:, 763 + number_of_quotes: extra_quotes + 3, 761 764 beginning_whitespace:, 762 765 lines:, 763 766 end_indentation:, ··· 766 769 } 767 770 } 768 771 772 + fn count_extra_quotes(lexer: Lexer, extra: Int) -> #(Lexer, Int) { 773 + case lexer.source { 774 + "\"" <> source -> count_extra_quotes(advance(lexer, source), extra + 1) 775 + _ -> #(lexer, extra) 776 + } 777 + } 778 + 769 779 fn is_whitespace(string: String) -> Bool { 770 780 case string { 771 781 "" -> True ··· 801 811 lexer: Lexer, 802 812 lines: List(String), 803 813 current_line: String, 814 + extra_quotes: Int, 804 815 ) -> #(Lexer, List(String), String) { 805 816 let #(before, split, after) = 806 817 splitter.split(lexer.splitters.triple_quoted_string, lexer.source) ··· 808 819 let before = current_line <> before 809 820 810 821 case split { 811 - "\"\"\"" -> 822 + "\"\"\"" -> { 823 + let lexer = advance(lexer, after) 812 824 case is_whitespace(before) { 813 825 False -> 814 826 lex_triple_quoted_string_contents( 815 - advance(lexer, after), 827 + lexer, 816 828 lines, 817 829 before <> "\"\"\"", 830 + extra_quotes, 818 831 ) 819 - True -> #(advance(lexer, after), lines, before) 832 + True if extra_quotes == 0 -> #(lexer, lines, before) 833 + True -> 834 + case consume_extra_quotes(lexer, extra_quotes) { 835 + Ok(lexer) -> #(lexer, lines, before) 836 + Error(Nil) -> 837 + lex_triple_quoted_string_contents( 838 + lexer, 839 + lines, 840 + before <> "\"\"\"", 841 + extra_quotes, 842 + ) 843 + } 820 844 } 845 + } 821 846 822 847 "\n" | "\r\n" -> 823 848 lex_triple_quoted_string_contents( 824 849 advance(lexer, after), 825 850 [before, ..lines], 826 851 "", 852 + extra_quotes, 827 853 ) 828 854 829 855 _ -> #(error(lexer, UnterminatedString), [before, ..lines], "") 856 + } 857 + } 858 + 859 + fn consume_extra_quotes(lexer: Lexer, extra_quotes: Int) -> Result(Lexer, Nil) { 860 + case extra_quotes, lexer.source { 861 + 0, _ -> Ok(lexer) 862 + _, "\"" <> source -> 863 + consume_extra_quotes(advance(lexer, source), extra_quotes - 1) 864 + _, _ -> Error(Nil) 830 865 } 831 866 } 832 867
+10 -3
src/pearl/token.gleam
··· 17 17 String(String) 18 18 TripleQuotedString( 19 19 sigil: option.Option(String), 20 + number_of_quotes: Int, 20 21 beginning_whitespace: String, 21 22 lines: List(String), 22 23 end_indentation: String, ··· 125 126 Atom(name:, quoted: True) -> "'" <> name <> "'" 126 127 Atom(name:, quoted: False) -> name 127 128 String(contents) -> "\"" <> contents <> "\"" 128 - TripleQuotedString(sigil:, beginning_whitespace:, lines:, end_indentation:) -> 129 + TripleQuotedString( 130 + sigil:, 131 + number_of_quotes:, 132 + beginning_whitespace:, 133 + lines:, 134 + end_indentation:, 135 + ) -> 129 136 case sigil { 130 137 option.None -> "" 131 138 option.Some(sigil) -> "~" <> sigil 132 139 } 133 - <> "\"\"\"" 140 + <> string.repeat("\"", number_of_quotes) 134 141 <> beginning_whitespace 135 142 <> string.join( 136 143 list.map(lines, fn(line) { end_indentation <> line }), ··· 138 145 ) 139 146 <> "\n" 140 147 <> end_indentation 141 - <> "\"\"\"" 148 + <> string.repeat("\"", number_of_quotes) 142 149 Sigil(sigil:, delimiter:, contents:) -> { 143 150 let #(opening, closing) = sigil_delimiters(delimiter) 144 151 "~" <> sigil <> opening <> contents <> closing
+23
test/pearl_test.gleam
··· 109 109 assert_tokens(src, [ 110 110 token.TripleQuotedString( 111 111 sigil: None, 112 + number_of_quotes: 3, 112 113 beginning_whitespace: " \n", 113 114 lines: [ 114 115 "Hello, this is triple-quoted!", ··· 132 133 assert_tokens(src, [ 133 134 token.TripleQuotedString( 134 135 sigil: Some("b"), 136 + number_of_quotes: 3, 135 137 beginning_whitespace: "\n", 136 138 lines: ["Hello", "This is a triple-quoted sigil"], 137 139 end_indentation: " ", 140 + ), 141 + ]) 142 + } 143 + 144 + pub fn triple_quoted_string_more_quotes_test() { 145 + let src = 146 + " 147 + \"\"\"\"\" 148 + This string has five quotes 149 + so four are allowed: 150 + \"\"\"\" 151 + \"\"\"\"\" 152 + " 153 + 154 + assert_tokens(src, [ 155 + token.TripleQuotedString( 156 + sigil: None, 157 + number_of_quotes: 5, 158 + beginning_whitespace: "\n", 159 + lines: ["This string has five quotes", "so four are allowed:", "\"\"\"\""], 160 + end_indentation: " ", 138 161 ), 139 162 ]) 140 163 }
+12
test/tokens.txt
··· 25 25 bye 26 26 """ 27 27 28 + """""" 29 + This string has six (6) quotes! 30 + That means you can have five quotes just fine: 31 + """"" 32 + And it's still valid. Six can also be here: """""" 33 + """""" 34 + 35 + ~"""" 36 + Here's a sigil with more than 3 quotes 37 + It works fine 38 + """" 39 + 28 40 after 29 41 begin 30 42 case