lexer.spice

// Std imports
import "std/data/vector";
import "std/text/format";
import "std/text/analysis";

// Own imports
//import "../source-file";
//import "../compiler-pass";
import "../lexer/token";
import "../reader/reader";
import "../reader/code-loc";

public type Lexer struct/* : ICompilerPass*/ {
    //compose CompilerPass compilerPass
    Reader reader
    Token curTok
}

public p Lexer.ctor(string filePath) {
    this.reader = Reader(filePath);
    this.curTok = Token(TokenType::INVALID);

    // Read and consume first token
    this.advance();
}

/*public p Lexer.ctor(SourceFile* sourceFile) {
    this.ctor(sourceFile.filePath);
}*/

public f<const Token&> Lexer.getToken() {
    return this.curTok;
}

public p Lexer.advance() {
    // Skip any whitespaces
    while (isWhitespace(this.reader.getChar()) && !this.reader.isEOF()) {
        this.reader.advance();
    }

    // Read and consume next token
    this.curTok = this.consumeToken();
}

public p Lexer.expect(TokenType expectedType) {
    if (this.curTok.tokenType != expectedType) {
        panic(Error("The type of the current token does not match the expected type"));
    }
    this.advance();
}

public p Lexer.expectOneOf(Vector<TokenType> expectedTypes) {
    foreach TokenType expectedType : expectedTypes.getIterator() {
        if (this.curTok.tokenType == expectedType) {
            return;
        }
    }
    panic(Error("The type of the current token was not amongst the expected types"));
}

public f<bool> Lexer.isEOF() {
    return this.curTok.tokenType == TokenType::EOF;
}

public f<CodeLoc> Lexer.getCodeLoc() {
    return this.curTok.codeLoc;
}

f<Token> Lexer.consumeToken() {
    // Get the current char from the reader instance
    char curChar = this.reader.getChar();
    const CodeLoc codeLoc = this.reader.getCodeLoc();

    // Check if EOF
    if this.reader.isEOF() {
        return Token(TokenType::EOF, "EOF", codeLoc);
    }
    // Check if identifier
    if isAlpha(curChar) || curChar == '_' {
        if isUpper(curChar) { // Type identifier
            return this.consumeTypeIdentifier(codeLoc);
        } else { // Normal identifier or keyword
            return this.consumeKeywordOrIdentifier(codeLoc);
        }
    }
    // Check if number
    if isDigit(curChar) {
        // Consume number literal
        Token numericLiteral = this.consumeNumberLiteral(codeLoc);

        // Patch token type if suffix is present
        const char literalSuffix = this.reader.getChar();
        if literalSuffix == 's' {
            numericLiteral.tokenType = TokenType::SHORT_LIT;
        } else if literalSuffix == 'l' {
            numericLiteral.tokenType = TokenType::LONG_LIT;
        }

        return numericLiteral;
    }
    // Check if char literal
    if curChar == '\'' {
        return this.consumeCharLiteral(codeLoc);
    }
    // Check if string literal
    if curChar == '"' {
        return this.consumeStringLiteral(codeLoc);
    }

    // Check if operator can consumed
    if curChar == '{' {
        this.reader.advance(); // Consume '{'
        return Token(TokenType::LBRACE, "{", codeLoc);
    }
    if curChar == '}' {
        this.reader.advance(); // Consume '}'
        return Token(TokenType::RBRACE, "}", codeLoc);
    }
    if curChar == '(' {
        this.reader.advance(); // Consume '('
        return Token(TokenType::LPAREN, "(", codeLoc);
    }
    if curChar == ')' {
        this.reader.advance(); // Consume ')'
        return Token(TokenType::RPAREN, ")", codeLoc);
    }
    if curChar == '[' {
        this.reader.advance(); // Consume '['
        return Token(TokenType::LBRACKET, "[", codeLoc);
    }
    if curChar == ']' {
        this.reader.advance(); // Consume ']'
        return Token(TokenType::RBRACKET, "]", codeLoc);
    }
    if curChar == '|' {
        this.reader.advance(); // Consume '|'
        curChar = this.reader.getChar();
        if this.reader.getChar() == '|' { // "||"
            this.reader.advance(); // Consume '|'
            return Token(TokenType::LOGICAL_OR, "||", codeLoc);
        } else { // '|'
            return Token(TokenType::BITWISE_OR, "||", codeLoc);
        }
    }
    if curChar == '&' {
        this.reader.advance(); // Consume '&'
        curChar = this.reader.getChar();
        if this.reader.getChar() == '&' { // "&&"
            this.reader.advance(); // Consume '&'
            return Token(TokenType::LOGICAL_AND, "&&", codeLoc);
        } else { // '&'
            return Token(TokenType::BITWISE_AND, "&&", codeLoc);
        }
    }
    if curChar == '^' {
        this.reader.advance(); // Consume '^'
        if this.reader.getChar() == '=' { // "^="
            this.reader.advance(); // Consume '='
            return Token(TokenType::XOR_EQUAL, "^=", codeLoc);
        } else { // '^'
            return Token(TokenType::BITWISE_XOR, "^", codeLoc);
        }
    }
    if curChar == '~' {
        this.reader.advance(); // Consume '~'
        return Token(TokenType::BITWISE_NOT, "~", codeLoc);
    }
    if curChar == '!' {
        this.reader.advance(); // Consume '!'
        if this.reader.getChar() == '=' { // "!="
            this.reader.advance(); // Consume '='
            return Token(TokenType::NOT_EQUAL, "!=", codeLoc);
        } else { // '!'
            return Token(TokenType::NOT, "!", codeLoc);
        }
    }
    if curChar == '+' {
        this.reader.advance(); // Consume '+'
        if this.reader.getChar() == '+' { // "++"
            this.reader.advance(); // Consume '+'
            return Token(TokenType::PLUS_PLUS, "++", codeLoc);
        } else if this.reader.getChar() == '=' { // "+="
            this.reader.advance(); // Consume '='
            return Token(TokenType::PLUS_EQUAL, "+=", codeLoc);
        } else { // '+'
            return Token(TokenType::PLUS, "+", codeLoc);
        }
    }
    if curChar == '-' {
        this.reader.advance(); // Consume '-'
        if this.reader.getChar() == '-' { // "--"
            this.reader.advance(); // Consume '-'
            return Token(TokenType::MINUS_MINUS, "--", codeLoc);
        } else if this.reader.getChar() == '=' { // "-="
            this.reader.advance(); // Consume '='
            return Token(TokenType::MINUS_EQUAL, "-=", codeLoc);
        } else if this.reader.getChar() == '>' { // "->"
            this.reader.advance(); // Consume '>'
            return Token(TokenType::ARROW, "->", codeLoc);
        } else { // '-'
            return Token(TokenType::MINUS, "-", codeLoc);
        }
    }
    if curChar == '*' {
        this.reader.advance(); // Consume '*'
        if this.reader.getChar() == '=' { // "*="
            this.reader.advance(); // Consume '='
            return Token(TokenType::MUL_EQUAL, "*=", codeLoc);
        } else { // '*'
            return Token(TokenType::MUL, "*", codeLoc);
        }
    }
    if curChar == '/' {
        this.reader.advance(); // Consume '/'
        if this.reader.getChar() == '=' { // "/="
            this.reader.advance(); // Consume '='
            return Token(TokenType::DIV_EQUAL, "/=", codeLoc);
        } else if this.reader.getChar() == '/' { // "//"
            return this.consumeLineComment(codeLoc);
        } else if this.reader.getChar() == '*' { // "/*"
            return this.consumeBlockOrDocComment(codeLoc);
        } else { // '/'
            return Token(TokenType::DIV, "/", codeLoc);
        }
    }
    if curChar == '%' {
        this.reader.advance(); // Consume '%'
        if this.reader.getChar() == '=' { // "%="
            this.reader.advance(); // Consume '='
            return Token(TokenType::REM_EQUAL, "%=", codeLoc);
        } else { // '%'
            return Token(TokenType::REM, "%", codeLoc);
        }
    }
    if curChar == '=' {
        this.reader.advance(); // Consume '='
        curChar = this.reader.getChar();
        if this.reader.getChar() == '=' { // "=="
            this.reader.advance(); // Consume '='
            return Token(TokenType::EQUAL, "==", codeLoc);
        } else { // '='
            return Token(TokenType::ASSIGN, "=", codeLoc);
        }
    }
    if curChar == '<' {
        this.reader.advance(); // Consume '<'
        curChar = this.reader.getChar();
        if this.reader.getChar() == '=' { // "<="
            this.reader.advance(); // Consume '='
            return Token(TokenType::LESS_EQUAL, "<=", codeLoc);
        } else if this.reader.getChar() == '<' { // "<<"
            this.reader.advance(); // Consume '<'
            if this.reader.getChar() == '=' { // "<<="
                this.reader.advance(); // Consume '='
                return Token(TokenType::SHL_EQUAL, "<<=", codeLoc);
            } else { // "<<"
                return Token(TokenType::SHL, "<<", codeLoc);
            }
        } else { // '<'
            return Token(TokenType::LESS, "<", codeLoc);
        }
    }
    if curChar == '>' {
        this.reader.advance(); // Consume '>'
        curChar = this.reader.getChar();
        if this.reader.getChar() == '=' { // ">="
            this.reader.advance(); // Consume '='
            return Token(TokenType::GREATER_EQUAL, ">=", codeLoc);
        } else if this.reader.getChar() == '>' { // ">>"
            this.reader.advance(); // Consume '>'
            if this.reader.getChar() == '=' { // ">>="
                this.reader.advance(); // Consume '='
                return Token(TokenType::SHR_EQUAL, ">>=", codeLoc);
            } else { // ">>"
                return Token(TokenType::SHR, ">>", codeLoc);
            }
        } else { // '>'
            return Token(TokenType::GREATER, ">", codeLoc);
        }
    }
    if curChar == '?' {
        this.reader.advance(); // Consume '?'
        return Token(TokenType::QUESTION_MARK, "?", codeLoc);
    }
    if curChar == ':' {
        this.reader.advance(); // Consume ':'
        if this.reader.getChar() == ':' { // "::"
            this.reader.advance(); // Consume ':'
            return Token(TokenType::SCOPE_ACCESS, "::", codeLoc);
        } else { // ':'
            return Token(TokenType::COLON, ":", codeLoc);
        }
    }
    if curChar == ';' {
        this.reader.advance(); // Consume ';'
        return Token(TokenType::SEMICOLON, ";", codeLoc);
    }
    if curChar == ',' {
        this.reader.advance(); // Consume ','
        return Token(TokenType::COMMA, ",", codeLoc);
    }
    if curChar == '.' {
        this.reader.advance(); // Consume '.'
        if this.reader.getChar() == '.' { // ".."
            this.reader.advance(); // Consume second '.'
            this.reader.expect('.'); // Consume third '.'
            return Token(TokenType::ELLIPSIS, "...", codeLoc);
        } else { // '.'
            return Token(TokenType::DOT, ".", codeLoc);
        }
    }
    if curChar == '#' {
        this.reader.advance(); // Consume '#'
        if this.reader.getChar() == '!' { // "#!"
            this.reader.advance(); // Consume '!'
            return Token(TokenType::MOD_ATTR_PREAMBLE, "#!", codeLoc);
        } else { // '#'
            return Token(TokenType::FCT_ATTR_PREAMBLE, "#", codeLoc);
        }
    }

    panic(Error("Got unexpected character"));
}

f<Token> Lexer.consumeNumberLiteral(const CodeLoc& codeLoc) {
    String numberStr;

    // Check for optional minus sign
    if this.reader.getChar() == '-' {
        numberStr += '-';
        this.reader.advance(); // Consume '-'
    }

    // Check for different numeric literal formats
    if this.reader.getChar() == '0' { // With base prefix
        this.reader.advance(); // Consume '0'
        numberStr += '0';
        // Read and consumenext char, which is the base of the number ('x', 'b', 'o' or 'd')
        const char base = toLower(this.reader.getChar());
        numberStr += this.reader.getChar();
        this.reader.advance();
        // Decide what to do, depending on the base
        if base == 'x' { // Hexadecimal number
            this.reader.advance(); // Consume 'x'
            while isHexDigit(this.reader.getChar()) {
                numberStr += this.reader.getChar();
                this.reader.advance();
            }
        } else if base == 'b' { // Binary number
            this.reader.advance(); // Consume 'b'
            while isBinDigit(this.reader.getChar()) {
                numberStr += this.reader.getChar();
                this.reader.advance();
            }
        } else if base == 'o' { // Octal number
            this.reader.advance(); // Consume 'o'
            while isOctDigit(this.reader.getChar()) {
                numberStr += this.reader.getChar();
                this.reader.advance();
            }
        } else { // Decimal number
            if this.reader.getChar() == 'd' {
                this.reader.advance(); // Consume 'd'
            }
            while isDigit(this.reader.getChar()) {
                numberStr += this.reader.getChar();
                this.reader.advance();
            }
        }
    } else { // Decimal number
        while isDigit(this.reader.getChar()) {
            numberStr += this.reader.getChar();
            this.reader.advance();
        }
    }

    // The correct token type is set a layer above in the parsing functions for int, short, long, etc.
    return Token(TokenType::INT_LIT, numberStr, codeLoc);
}

f<Token> Lexer.consumeCharLiteral(const CodeLoc& codeLoc) {
    // Parse the following ANTLR regex: '\'' (~['\\\r\n] | '\\' (. | EOF)) '\''
    String charStr;
    this.reader.expect('\''); // Consume '\''
    if this.reader.getChar() == '\\' { // Escape sequence
        this.reader.advance(); // Consume '\\'
        const char nextChar = this.reader.getChar();
        if nextChar == 'n' || nextChar == 'r' || nextChar == 't' || nextChar == '0' || nextChar == '\'' || nextChar == '"' || nextChar == '\\' {
            charStr += nextChar;
        } else {
            panic(Error("Invalid escape sequence"));
        }
        this.reader.advance(); // Consume escaped character
    } else { // Normal character
        charStr += this.reader.getChar();
        this.reader.advance(); // Consume character
    }
    this.reader.expect('\''); // Consume '\''
    return Token(TokenType::CHAR_LIT, charStr, codeLoc);
}

f<Token> Lexer.consumeStringLiteral(const CodeLoc& codeLoc) {
    // Parse the following ANTLR regex: '"' (~["\\\r\n] | '\\' (. | EOF))* '"'
    String stringStr;
    this.reader.expect('"'); // Consume '"'
    while this.reader.getChar() != '"' {
        if this.reader.getChar() == '\\' { // Escape sequence
            this.reader.advance(); // Consume '\\'
            const char nextChar = this.reader.getChar();
            if nextChar == 'n' || nextChar == 'r' || nextChar == 't' || nextChar == '0' || nextChar == '\'' || nextChar == '"' || nextChar == '\\' {
                stringStr += nextChar;
            } else {
                panic(Error("Invalid escape sequence"));
            }
            this.reader.advance(); // Consume escaped character
        } else { // Normal character
            stringStr += this.reader.getChar();
            this.reader.advance(); // Consume character
        }
    }
    this.reader.expect('"'); // Consume '"'
    return Token(TokenType::STRING_LIT, stringStr, codeLoc);
}

f<Token> Lexer.consumeKeywordOrIdentifier(const CodeLoc& codeLoc) {
    String identifier;
    do {
        identifier += this.reader.getChar();
        this.reader.advance();
    } while isAlphaNum(this.reader.getChar()) || this.reader.getChar() == '_';

    // Check if identifier is a keyword
    if identifier == "double" {
        return Token(TokenType::TYPE_DOUBLE, "double", codeLoc);
    }
    if identifier == "int" {
        return Token(TokenType::TYPE_INT, "int", codeLoc);
    }
    if identifier == "short" {
        return Token(TokenType::TYPE_SHORT, "short", codeLoc);
    }
    if identifier == "long" {
        return Token(TokenType::TYPE_LONG, "long", codeLoc);
    }
    if identifier == "byte" {
        return Token(TokenType::TYPE_BYTE, "byte", codeLoc);
    }
    if identifier == "char" {
        return Token(TokenType::TYPE_CHAR, "char", codeLoc);
    }
    if identifier == "string" {
        return Token(TokenType::TYPE_STRING, "string", codeLoc);
    }
    if identifier == "bool" {
        return Token(TokenType::TYPE_BOOL, "bool", codeLoc);
    }
    if identifier == "dyn" {
        return Token(TokenType::TYPE_DYN, "dyn", codeLoc);
    }
    if identifier == "const" {
        return Token(TokenType::CONST, "const", codeLoc);
    }
    if identifier == "signed" {
        return Token(TokenType::SIGNED, "signed", codeLoc);
    }
    if identifier == "unsigned" {
        return Token(TokenType::UNSIGNED, "unsigned", codeLoc);
    }
    if identifier == "inline" {
        return Token(TokenType::INLINE, "inline", codeLoc);
    }
    if identifier == "public" {
        return Token(TokenType::PUBLIC, "public", codeLoc);
    }
    if identifier == "heap" {
        return Token(TokenType::HEAP, "heap", codeLoc);
    }
    if identifier == "compose" {
        return Token(TokenType::COMPOSE, "compose", codeLoc);
    }
    if identifier == "f" {
        return Token(TokenType::F, "f", codeLoc);
    }
    if identifier == "p" {
        return Token(TokenType::P, "p", codeLoc);
    }
    if identifier == "if" {
        return Token(TokenType::IF, "if", codeLoc);
    }
    if identifier == "else" {
        return Token(TokenType::ELSE, "else", codeLoc);
    }
    if identifier == "assert" {
        return Token(TokenType::ASSERT, "assert", codeLoc);
    }
    if identifier == "for" {
        return Token(TokenType::FOR, "for", codeLoc);
    }
    if identifier == "foreach" {
        return Token(TokenType::FOREACH, "foreach", codeLoc);
    }
    if identifier == "do" {
        return Token(TokenType::DO, "do", codeLoc);
    }
    if identifier == "while" {
        return Token(TokenType::WHILE, "while", codeLoc);
    }
    if identifier == "import" {
        return Token(TokenType::IMPORT, "import", codeLoc);
    }
    if identifier == "break" {
        return Token(TokenType::BREAK, "break", codeLoc);
    }
    if identifier == "continue" {
        return Token(TokenType::CONTINUE, "continue", codeLoc);
    }
    if identifier == "return" {
        return Token(TokenType::RETURN, "return", codeLoc);
    }
    if identifier == "as" {
        return Token(TokenType::AS, "as", codeLoc);
    }
    if identifier == "struct" {
        return Token(TokenType::STRUCT, "struct", codeLoc);
    }
    if identifier == "interface" {
        return Token(TokenType::INTERFACE, "interface", codeLoc);
    }
    if identifier == "type" {
        return Token(TokenType::TYPE, "type", codeLoc);
    }
    if identifier == "enum" {
        return Token(TokenType::ENUM, "enum", codeLoc);
    }
    if identifier == "operator" {
        return Token(TokenType::OPERATOR, "operator", codeLoc);
    }
    if identifier == "alias" {
        return Token(TokenType::ALIAS, "alias", codeLoc);
    }
    if identifier == "unsafe" {
        return Token(TokenType::UNSAFE, "unsafe", codeLoc);
    }
    if identifier == "nil" {
        return Token(TokenType::NIL, "nil", codeLoc);
    }
    if identifier == "main" {
        return Token(TokenType::MAIN, "main", codeLoc);
    }
    if identifier == "printf" {
        return Token(TokenType::PRINTF, "printf", codeLoc);
    }
    if identifier == "sizeof" {
        return Token(TokenType::SIZEOF, "sizeof", codeLoc);
    }
    if identifier == "alignof" {
        return Token(TokenType::ALIGNOF, "alignof", codeLoc);
    }
    if identifier == "len" {
        return Token(TokenType::LEN, "len", codeLoc);
    }
    if identifier == "panic" {
        return Token(TokenType::PANIC, "panic", codeLoc);
    }
    if identifier == "ext" {
        return Token(TokenType::EXT, "ext", codeLoc);
    }
    if identifier == "true" {
        return Token(TokenType::TRUE, "true", codeLoc);
    }
    if identifier == "false" {
        return Token(TokenType::FALSE, "false", codeLoc);
    }

    // No keyword was matched -> treat as identifier
    return Token(TokenType::IDENTIFIER, identifier, codeLoc);
}

f<Token> Lexer.consumeTypeIdentifier(const CodeLoc& codeLoc) {
    String identifier;
    do {
        identifier += this.reader.getChar();
        this.reader.advance();
    } while isAlphaNum(this.reader.getChar()) || this.reader.getChar() == '_';
    return Token(TokenType::TYPE_IDENTIFIER, identifier, codeLoc);
}

f<Token> Lexer.consumeLineComment(const CodeLoc& codeLoc) {
    this.reader.expect('/'); // Consume second '/' (first '/' is consumed by the caller)

    String comment = String("//");
    while !this.reader.isEOF() {
        // Check for comment end
        if this.reader.getChar() == '\n' {
            this.reader.advance(); // Consume '\n'
            break;
        }

        comment += this.reader.getChar();
        this.reader.advance();
    }

    return Token(TokenType::LINE_COMMENT, comment, codeLoc);
}

f<Token> Lexer.consumeBlockOrDocComment(const CodeLoc& codeLoc) {
    this.reader.expect('*'); // Consume '*' ('/' is consumed by the caller)
    String comment = String("/*");

    // Check if doc comment
    bool isDocComment = false;
    if this.reader.getChar() == '*' {
        isDocComment = true;
        this.reader.advance(); // Consume '*'
        comment += '*';
    }

    while !this.reader.isEOF() {
        // Check for comment end
        if this.reader.getChar() == '*' {
            this.reader.advance(); // Consume '*'
            comment += '*';
            if this.reader.getChar() == '/' {
                this.reader.advance(); // Consume '/'
                comment += '/';
                break;
            }
        }

        comment += this.reader.getChar();
        this.reader.advance();
    }

    return Token(isDocComment ? TokenType::DOC_COMMENT : TokenType::BLOCK_COMMENT, comment, codeLoc);
}

f<bool> Lexer.isCurrentTokenIgnored() {
    return this.curTok.tokenType == TokenType::LINE_COMMENT || this.curTok.tokenType == TokenType::BLOCK_COMMENT || this.curTok.tokenType == TokenType::DOC_COMMENT;
}