Add basic support for triple-quoted strings

src/pearl.gleam

···
     brace_escape_sequence: Splitter,
     sigil: Splitter,
     sigil_verbatim: Splitter,
+    triple_quoted_string: Splitter,
   )
 }
 
···
   UnterminatedCharacter
   UnterminatedEscapeSequence
   ExpectedSigilDelimiter
+  ExpectedWhitespaceAfterTripleQuote
+  InvalidTripleQuotedStringIndentation(
+    expected_indentation: String,
+    line: String,
+  )
 }
 
 pub fn new(source: String) -> Lexer {
···
 
 fn make_splitters() -> Splitters {
   Splitters(
-    until_end_of_line: splitter.new(["\n", "\r"]),
+    until_end_of_line: splitter.new(["\n", "\r\n"]),
     string: splitter.new(["\"", "\\"]),
     quoted_atom: splitter.new(["'", "\\"]),
-    brace_escape_sequence: splitter.new(["}", "\n", "\r"]),
+    brace_escape_sequence: splitter.new(["}", "\n", "\r\n"]),
     sigil: splitter.new([
       ")", "]", "}", ">", "/", "|", "'", "\"", "`", "#", "\\",
     ]),
     sigil_verbatim: splitter.new([
       ")", "]", "}", ">", "/", "|", "'", "\"", "`", "#",
     ]),
+    triple_quoted_string: splitter.new(["\n", "\r\n", "\"\"\""]),
   )
 }
 
···
     | "8" as char <> source
     | "9" as char <> source ->
       lex_number(advance(lexer, source), char, Initial, AfterNumber)
+
+    "\"\"\"" <> source -> lex_triple_quoted_string(advance(lexer, source))
 
     "\"" <> source -> lex_string(advance(lexer, source), "")
     "'" <> source -> lex_quoted_atom(advance(lexer, source), "")
···
     }
 
     _ -> #(advance(lexer, after), token.String(contents <> before))
+  }
+}
+
+fn lex_triple_quoted_string(lexer: Lexer) -> #(Lexer, Token) {
+  let #(lexer, beginning_whitespace) = case
+    splitter.split(lexer.splitters.until_end_of_line, lexer.source)
+  {
+    #(_, "", _) -> #(error(lexer, ExpectedWhitespaceAfterTripleQuote), "")
+    #(before, newline, after) ->
+      case is_whitespace(before) {
+        True -> #(advance(lexer, after), before <> newline)
+        False -> #(error(lexer, ExpectedWhitespaceAfterTripleQuote), "")
+      }
+  }
+
+  let #(lexer, lines, end_indentation) =
+    lex_triple_quoted_string_contents(lexer, [], "")
+
+  case strip_line_prefixes(lines, end_indentation, []) {
+    Error(line) -> {
+      let contents =
+        beginning_whitespace
+        <> string.join(list.reverse(lines), "\n")
+        <> "\n"
+        <> end_indentation
+      #(
+        error(
+          lexer,
+          InvalidTripleQuotedStringIndentation(
+            expected_indentation: end_indentation,
+            line:,
+          ),
+        ),
+        token.InvalidTripleQuotedString(contents),
+      )
+    }
+    Ok(lines) -> #(
+      lexer,
+      token.TripleQuotedString(beginning_whitespace:, lines:, end_indentation:),
+    )
+  }
+}
+
+fn is_whitespace(string: String) -> Bool {
+  case string {
+    "" -> True
+    " " <> string
+    | "\n" <> string
+    | "\r" <> string
+    | "\t" <> string
+    | "\f" <> string -> is_whitespace(string)
+    _ -> False
+  }
+}
+
+fn strip_line_prefixes(
+  lines: List(String),
+  end_indentation: String,
+  acc: List(String),
+) -> Result(List(String), String) {
+  case lines {
+    [] -> Ok(acc)
+    [line, ..lines] ->
+      case strip_prefix(line, end_indentation) {
+        Ok(line) -> strip_line_prefixes(lines, end_indentation, [line, ..acc])
+        Error(_) -> Error(line)
+      }
+  }
+}
+
+@external(erlang, "pearl_ffi", "strip_prefix")
+@external(javascript, "./pearl_ffi.mjs", "strip_prefix")
+fn strip_prefix(string: String, prefix: String) -> Result(String, Nil)
+
+fn lex_triple_quoted_string_contents(
+  lexer: Lexer,
+  lines: List(String),
+  current_line: String,
+) -> #(Lexer, List(String), String) {
+  let #(before, split, after) =
+    splitter.split(lexer.splitters.triple_quoted_string, lexer.source)
+
+  let before = current_line <> before
+
+  case split {
+    "\"\"\"" ->
+      case is_whitespace(before) {
+        False ->
+          lex_triple_quoted_string_contents(
+            advance(lexer, after),
+            lines,
+            before <> "\"\"\"",
+          )
+        True -> #(advance(lexer, after), lines, before)
+      }
+
+    "\n" | "\r\n" ->
+      lex_triple_quoted_string_contents(
+        advance(lexer, after),
+        [before, ..lines],
+        "",
+      )
+
+    _ -> #(error(lexer, UnterminatedString), [before, ..lines], "")
   }
 }
 

src/pearl/token.gleam

···
+import gleam/list
+import gleam/string
+
 pub type Token {
   // Whitespace and comments
   Whitespace(String)
···
   Float(String)
   Atom(name: String, quoted: Bool)
   String(String)
-  TripleQuotedString(contents: String, end_indentation: String)
+  TripleQuotedString(
+    beginning_whitespace: String,
+    lines: List(String),
+    end_indentation: String,
+  )
   Sigil(sigil: String, delimiter: SigilDelimiter, contents: String)
   Variable(String)
 
···
   UnterminatedString(String)
   UnterminatedSigil(sigil: String, delimiter: SigilDelimiter, contents: String)
   UnterminatedAtom(String)
+  InvalidTripleQuotedString(contents: String)
 }
 
 pub fn to_source(token: Token) -> String {
···
     Atom(name:, quoted: True) -> "'" <> name <> "'"
     Atom(name:, quoted: False) -> name
     String(contents) -> "\"" <> contents <> "\""
-    TripleQuotedString(contents:, end_indentation:) ->
-      "\"\"\"\n" <> contents <> "\n" <> end_indentation <> "\"\"\""
+    TripleQuotedString(beginning_whitespace:, lines:, end_indentation:) ->
+      "\"\"\""
+      <> beginning_whitespace
+      <> string.join(
+        list.map(lines, fn(line) { end_indentation <> line }),
+        "\n",
+      )
+      <> "\n"
+      <> end_indentation
+      <> "\"\"\""
     Sigil(sigil:, delimiter:, contents:) -> {
       let #(opening, closing) = sigil_delimiters(delimiter)
       "~" <> sigil <> opening <> contents <> closing
···
       "~" <> sigil <> opening <> contents
     }
     UnterminatedAtom(contents) -> "'" <> contents
+    InvalidTripleQuotedString(contents) -> "\"\"\"" <> contents <> "\"\"\""
   }
 }
 

src/pearl_ffi.erl

···
+-module(pearl_ffi).
+
+-export([strip_prefix/2]).
+
+strip_prefix(String, Prefix) ->
+    Prefix_size = byte_size(Prefix),
+
+    case Prefix == binary_part(String, 0, Prefix_size) of
+        true ->
+            String_size = byte_size(String),
+            {ok, binary_part(String, Prefix_size, String_size - Prefix_size)};
+        false ->
+            {error, nil}
+    end.

src/pearl_ffi.mjs

···
+import { Ok, Error } from "./gleam.mjs";
+
+export function strip_prefix(string, prefix) {
+  if (string.startsWith(prefix)) {
+    return new Ok(string.slice(prefix.length));
+  } else {
+    return new Error(undefined);
+  }
+}

test/pearl_test.gleam

···
   ])
 }
 
+pub fn triple_quoted_string_test() {
+  let src =
+    "
+\"\"\"  
+\t Hello, this is triple-quoted!
+\t  This line starts with a space
+\t Quotes are allowed: \"\", even three: \"\"\"
+\t \"\"\"
+"
+
+  assert_tokens(src, [
+    token.TripleQuotedString(
+      beginning_whitespace: "  \n",
+      lines: [
+        "Hello, this is triple-quoted!",
+        " This line starts with a space",
+        "Quotes are allowed: \"\", even three: \"\"\"",
+      ],
+      end_indentation: "\t ",
+    ),
+  ])
+}
+
 pub fn unknown_character_test() {
   let src = "a&b"
   assert_errors(src, [pearl.UnknownCharacter("&")])
···
   let src = "1.2e-"
   assert_errors(src, [pearl.ExpectedExponent])
 }
+
+pub fn missing_whitespace_after_triple_quote() {
+  let src =
+    "\"\"\"Hello
+\"\"\""
+  assert_errors(src, [pearl.ExpectedWhitespaceAfterTripleQuote])
+}
+
+pub fn invalid_triple_quoted_string_indentation() {
+  let src =
+    "\"\"\"
+ Hello
+    world
+   \"\"\""
+  assert_errors(src, [
+    pearl.InvalidTripleQuotedStringIndentation(
+      expected_indentation: "   ",
+      line: " Hello",
+    ),
+  ])
+}

test/tokens.txt

···
 ~/I need "quotes"/ ~<This is also possible> ~b"Bit array"
 Variable Something_else@5
 
+"""   
+  Triple quoted!
+    This line is indented, and has quotes: ""
+  Three quotes allowed here: """ and here: """
+  """
+
 after
 begin
 case