gott / lexer.go

package gott

import (
    "strings"
    "unicode"
    "unicode/utf8"
)

const eof = -1

// Lexer tokenizes a template string into a sequence of tokens
type Lexer struct {
    input   string     // the string being scanned
    start   int        // start position of current token
    pos     int        // current position in input
    width   int        // width of last rune read
    line    int        // current line number (1-based)
    linePos int        // position of start of current line
    tokens  chan Token // channel of scanned tokens
}

// stateFn represents a lexer state function; returns the next state
type stateFn func(*Lexer) stateFn

// NewLexer creates a new lexer for the given input and starts scanning
func NewLexer(input string) *Lexer {
    l := &Lexer{
        input:   input,
        line:    1,
        linePos: 0,
        tokens:  make(chan Token, 2),
    }
    go l.run()
    return l
}

// run executes the state machine
func (l *Lexer) run() {
    for state := lexText; state != nil; {
        state = state(l)
    }
    close(l.tokens)
}

// NextToken returns the next token from the lexer
func (l *Lexer) NextToken() Token {
    return <-l.tokens
}

// Tokens returns all tokens as a slice (consumes the channel)
func (l *Lexer) Tokens() []Token {
    var tokens []Token
    for tok := range l.tokens {
        tokens = append(tokens, tok)
    }
    return tokens
}

// emit sends a token to the tokens channel
func (l *Lexer) emit(t TokenType) {
    l.tokens <- Token{
        Type:  t,
        Value: l.input[l.start:l.pos],
        Pos:   l.currentPos(),
    }
    l.start = l.pos
}

// emitValue sends a token with a specific value
func (l *Lexer) emitValue(t TokenType, value string) {
    l.tokens <- Token{
        Type:  t,
        Value: value,
        Pos:   l.currentPos(),
    }
    l.start = l.pos
}

// currentPos returns the current position for error reporting
func (l *Lexer) currentPos() Position {
    return Position{
        Line:   l.line,
        Column: l.start - l.linePos + 1,
        Offset: l.start,
    }
}

// next returns the next rune in the input and advances the position
func (l *Lexer) next() rune {
    if l.pos >= len(l.input) {
        l.width = 0
        return eof
    }
    r, w := utf8.DecodeRuneInString(l.input[l.pos:])
    l.width = w
    l.pos += l.width
    if r == '\n' {
        l.line++
        l.linePos = l.pos
    }
    return r
}

// backup steps back one rune (can only be called once per next call)
func (l *Lexer) backup() {
    l.pos -= l.width
    // If we backed up over a newline, decrement line count
    if l.pos < len(l.input) && l.input[l.pos] == '\n' {
        l.line--
        // Find previous line start
        l.linePos = strings.LastIndex(l.input[:l.pos], "\n") + 1
    }
}

// peek returns the next rune without advancing
func (l *Lexer) peek() rune {
    r := l.next()
    l.backup()
    return r
}

// ignore skips over the pending input
func (l *Lexer) ignore() {
    l.start = l.pos
}

// errorf emits an error token and terminates scanning
func (l *Lexer) errorf(format string, args ...any) stateFn {
    l.tokens <- Token{
        Type:  TokenError,
        Value: sprintf(format, args...),
        Pos:   l.currentPos(),
    }
    return nil
}

// sprintf is a simple formatter (avoiding fmt import in hot path)
func sprintf(format string, args ...any) string {
    // Simple implementation - for errors only
    result := format
    for _, arg := range args {
        if s, ok := arg.(string); ok {
            result = strings.Replace(result, "%s", s, 1)
            result = strings.Replace(result, "%q", "'"+s+"'", 1)
        }
    }
    return result
}

// skipWhitespace advances past any whitespace characters
func (l *Lexer) skipWhitespace() {
    for {
        r := l.next()
        if r == eof || !unicode.IsSpace(r) {
            l.backup()
            break
        }
    }
    l.ignore()
}

// hasPrefix checks if the remaining input starts with the given prefix
func (l *Lexer) hasPrefix(prefix string) bool {
    return strings.HasPrefix(l.input[l.pos:], prefix)
}

// ---- State Functions ----

// lexText scans text outside of tags until [% or EOF
func lexText(l *Lexer) stateFn {
    for {
        if l.hasPrefix("[%") {
            if l.pos > l.start {
                l.emit(TokenText)
            }
            return lexTagOpen
        }
        if l.next() == eof {
            break
        }
    }
    // Emit any remaining text
    if l.pos > l.start {
        l.emit(TokenText)
    }
    l.emit(TokenEOF)
    return nil
}

// lexTagOpen scans the [% opening delimiter
func lexTagOpen(l *Lexer) stateFn {
    l.pos += 2 // skip [%
    l.emit(TokenTagOpen)
    return lexInsideTag
}

// lexInsideTag scans inside a [% ... %] tag
func lexInsideTag(l *Lexer) stateFn {
    l.skipWhitespace()

    // Check for closing tag
    if l.hasPrefix("%]") {
        l.pos += 2
        l.emit(TokenTagClose)
        return lexText
    }

    // Check for two-character operators first
    twoCharOps := []struct {
        str string
        tok TokenType
    }{
        {"==", TokenEq},
        {"!=", TokenNe},
        {">=", TokenGe},
        {"<=", TokenLe},
        {"&&", TokenAnd},
        {"||", TokenOr},
    }
    for _, op := range twoCharOps {
        if l.hasPrefix(op.str) {
            l.pos += 2
            l.emit(op.tok)
            return lexInsideTag
        }
    }

    // Check for single-character operators/delimiters
    r := l.next()
    switch r {
    case eof:
        return l.errorf("unclosed tag")
    case '>':
        l.emit(TokenGt)
        return lexInsideTag
    case '<':
        l.emit(TokenLt)
        return lexInsideTag
    case '+':
        l.emit(TokenPlus)
        return lexInsideTag
    case '-':
        // Could be minus or negative number
        if unicode.IsDigit(l.peek()) {
            l.backup()
            return lexNumber
        }
        l.emit(TokenMinus)
        return lexInsideTag
    case '*':
        l.emit(TokenMul)
        return lexInsideTag
    case '/':
        l.emit(TokenDiv)
        return lexInsideTag
    case '%':
        // Check if this is %] (tag close) or % (modulo)
        if l.peek() == ']' {
            l.backup()
            l.pos += 2
            l.emit(TokenTagClose)
            return lexText
        }
        l.emit(TokenMod)
        return lexInsideTag
    case '.':
        l.emit(TokenDot)
        return lexInsideTag
    case '|':
        l.emit(TokenPipe)
        return lexInsideTag
    case '(':
        l.emit(TokenLParen)
        return lexInsideTag
    case ')':
        l.emit(TokenRParen)
        return lexInsideTag
    case ',':
        l.emit(TokenComma)
        return lexInsideTag
    case '=':
        l.emit(TokenAssign)
        return lexInsideTag
    case '$':
        l.emit(TokenDollar)
        return lexInsideTag
    case '"', '\'':
        l.backup()
        return lexString
    }

    // Check for number
    if unicode.IsDigit(r) {
        l.backup()
        return lexNumber
    }

    // Must be identifier or keyword
    if isAlpha(r) || r == '_' {
        l.backup()
        return lexIdentifier
    }

    return l.errorf("unexpected character: %s", string(r))
}

// lexString scans a quoted string literal (single or double quotes)
func lexString(l *Lexer) stateFn {
    quote := l.next() // consume opening quote
    l.ignore()        // don't include quote in value

    for {
        r := l.next()
        if r == eof {
            return l.errorf("unterminated string")
        }
        if r == '\\' {
            // Skip escaped character
            l.next()
            continue
        }
        if r == quote {
            // Don't include closing quote in value
            l.backup()
            l.emit(TokenString)
            l.next() // consume closing quote
            l.ignore()
            return lexInsideTag
        }
    }
}

// lexNumber scans a number (integer or float, possibly negative)
func lexNumber(l *Lexer) stateFn {
    // Optional leading minus
    if l.peek() == '-' {
        l.next()
    }

    // Integer part
    digits := false
    for unicode.IsDigit(l.peek()) {
        l.next()
        digits = true
    }

    if !digits {
        return l.errorf("expected digits in number")
    }

    // Optional decimal part
    if l.peek() == '.' {
        l.next()
        for unicode.IsDigit(l.peek()) {
            l.next()
        }
    }

    l.emit(TokenNumber)
    return lexInsideTag
}

// lexIdentifier scans an identifier or keyword
func lexIdentifier(l *Lexer) stateFn {
    for {
        r := l.next()
        if !isAlphaNumeric(r) && r != '_' {
            l.backup()
            break
        }
    }

    word := l.input[l.start:l.pos]
    tokType := LookupKeyword(word)
    l.emit(tokType)
    return lexInsideTag
}

// isAlpha returns true if r is an alphabetic character
func isAlpha(r rune) bool {
    return unicode.IsLetter(r)
}

// isAlphaNumeric returns true if r is alphanumeric
func isAlphaNumeric(r rune) bool {
    return unicode.IsLetter(r) || unicode.IsDigit(r)
}