Something went wrong. Try again.
Actually just three programming languages in a trenchcoat
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216use crate::syntax::{Amble, Document, SyntaxError};use crate::{Parse, Spanned, TokenPattern};use peekmore::{PeekMore, PeekMoreIterator};use trilogy_scanner::{Token, TokenType};
/// The parser for the Trilogy Programming Language.////// This parser takes a sequence of [`Token`][]s, typically from a [`Scanner`][trilogy_scanner::Scanner],/// and constructs it into an AST, which we call a [`Document`][].pub struct Parser<'src> { source: PeekMoreIterator<Box<dyn Iterator<Item = Token> + 'src>>, warnings: Vec<SyntaxError>, #[cfg(test)] // expose this thing to the test framework only pub(crate) errors: Vec<SyntaxError>, #[cfg(not(test))] errors: Vec<SyntaxError>, pub(crate) is_line_start: bool, pub(crate) is_spaced: bool,}
impl<'src> Parser<'src> { /// Construct a new parser taking input from an iterator of [`Token`][]s. The usual /// choice is to use a [`Scanner`][trilogy_scanner::Scanner] pub fn new<S: Iterator<Item = Token> + 'src>(source: S) -> Self { Self { source: (Box::new(source) as Box<dyn Iterator<Item = Token>>).peekmore(), errors: vec![], warnings: vec![], is_line_start: true, is_spaced: false, } }
/// Consume the tokens provided, attempting to build a [`Document`][] from them. /// /// Where possible, errors are recovered from and collected for later. The returned /// `Document` must not be used if the [`Parse`][] contains errors. pub fn parse(mut self) -> Parse<Document> { let ast = Amble::parse(&mut self); Parse { ast: ast.content, warnings: self.warnings, errors: self.errors, } }}
impl Parser<'_> { #[cfg_attr(not(feature = "lax"), allow(dead_code))] pub(crate) fn warn(&mut self, warning: SyntaxError) { self.warnings.push(warning); }
pub(crate) fn error(&mut self, error: SyntaxError) { self.errors.push(error); }
pub(crate) fn expected( &mut self, token: Token, message: impl std::fmt::Display, ) -> SyntaxError { let error = SyntaxError::new(token.span, message); self.error(error.clone()); error }
pub(crate) fn chomp(&mut self) { let mut invalid_tokens = vec![]; loop { let token = self.source.peek().expect("Peeked too many tokens"); if [ TokenType::EndOfLine, TokenType::CommentBlock, TokenType::CommentLine, TokenType::CommentInline, TokenType::Space, ] .matches(token) { self.next(); continue; } if TokenType::Error.matches(token) { invalid_tokens.push(self.next()); continue; } break; } if !invalid_tokens.is_empty() { self.error(SyntaxError::new( invalid_tokens.span(), "invalid characters in input", )); } }
fn peek_chomp(&mut self) { loop { let Some(token) = self.source.peek() else { return; }; if token.token_type == TokenType::EndOfFile { return; } if [ TokenType::EndOfLine, TokenType::CommentBlock, TokenType::CommentLine, TokenType::CommentInline, TokenType::Space, TokenType::Error, ] .matches(token) { self.source.advance_cursor(); continue; } break; } }
fn next(&mut self) -> Token { // Technically probably shouldn't unwrap here but if we consume the EndOfFile // it has to be at the end, at which point we consume no more, so this should // be safe. let token = self.source.next().expect("Consumed too many tokens");
#[rustfmt::skip] { use TokenType::*; // Different types of whitespace imply that we are truly at the start of a line // without any leading (non-whitespace) characters, as opposed to only the first // whole token on a line but other partial tokens were on this line already // (specifically, block comments). // // The ByteOrderMark, while not technically whitespace (or even allowed in most // parts of the code, for that matter) is included here because its presence is // not considered at all, so should not change the initial states of these bits // in much the same way that StartOfFile does not change them. // // That said, cases where line endings and startings are needed are uncertain, // maybe I don't need both of these flags. self.is_line_start = [EndOfLine, CommentLine, DocInner, DocOuter, ByteOrderMark, StartOfFile].matches(&token) || self.is_line_start && [CommentInline, Space].matches(&token); self.is_spaced = [EndOfLine, CommentLine, DocInner, DocOuter, CommentInline, CommentBlock, Space].matches(&token); }; token }
fn peek_next(&mut self) -> Option<Token> { self.peek_chomp(); let peeked = self.source.peek().cloned(); self.source.advance_cursor(); peeked }
pub(crate) fn expect_bang_oparen(&mut self) -> Result<(Token, Token), Token> { use TokenType::*; // Though tokenized as two tokens, this is kind of treated as one token as we // require `!(` to be unspaced in procedure calls. Since whitespace is a token, // a low-level peekmore after the high-level peek will sufficiently detect this. let next = self.peek().clone(); let after = self.source.peek_nth(1); if next.token_type == OpBang && after.unwrap().token_type == OParen { let bang = self.expect(OpBang).unwrap(); let oparen = self.expect(OParen).unwrap(); Ok((bang, oparen)) } else { Err(next) } }
pub(crate) fn peek(&mut self) -> &Token { self.chomp(); self.source.peek().unwrap() }
pub(crate) fn force_peek(&mut self) -> &Token { self.source.peek().unwrap() }
pub(crate) fn peekn(&mut self, n: usize) -> Option<Vec<Token>> { self.chomp(); let tokens = (0..n).map(|_| self.peek_next()).collect::<Option<Vec<_>>>(); self.source.reset_cursor(); tokens }
pub(crate) fn synchronize(&mut self, pattern: impl TokenPattern) { while !pattern.matches(self.peek()) { self.next(); } }
pub(crate) fn expect(&mut self, pattern: impl TokenPattern) -> Result<Token, Token> { let token = self.peek(); if !pattern.matches(token) { return Err(token.clone()); } Ok(self.next()) }
pub(crate) fn consume(&mut self) -> Token { self.chomp(); self.next() }
pub(crate) fn check(&mut self, pattern: impl TokenPattern) -> Result<&Token, &Token> { let token = self.peek(); if pattern.matches(token) { Ok(token) } else { Err(token) } }}