Token.h

Go to the documentation of this file.

#pragma once
#include "Location.h"
#include "sapphirepch.h"
 
/* All current token types. Will change in the future. */
enum TokenType
{
    TT_Identifier,
    TT_Number,
    TT_Float,
    TT_Keyword,
    TT_String,
    TT_Plus,
    TT_Minus,
    TT_Mul,
    TT_Div,
    TT_At,
    TT_Equal,
    TT_LessThan,
    TT_GreaterThan,
    TT_LParen,
    TT_RParen,
    TT_LBracket,
    TT_RBracket,
    TT_Semicolon,
    TT_LoadedString,
    TT_EOF,
    TT_Null,
    TT_Equals,
    TT_GTE,
    TT_LTE,
    TT_Period,
    TT_Comma,
    TT_Path,
    TT_Exclamation,
    TT_NEqual,
    TT_LSQB,
    TT_RSQB,
    TT_Type,
    TT_Import,
    TT_Syscall0,
    TT_Syscall1,
    TT_Syscall2,
    TT_Syscall3,
    TT_Syscall4,
    TT_Syscall5,
    TT_CompilerMacro
};
 
extern const std::string token_strings[];
 
/* Struct to represent tokens generated by the Lexer. */
struct Token
{
    TokenType tk_type;
 
    int int_value;
    std::string string_value;
    float float_value;
 
    Location loc;
 
    Token(const TokenType& type);
 
    Token(const TokenType& type, const Location& location);
 
    Token(const TokenType& type, const int& val, const Location& location);
 
    Token(const TokenType& type, const std::string& val, const Location& location);
 
    Token(const TokenType& type, const std::string& val);
 
    Token(const TokenType& type, const float& val, const Location& location);
 
    ~Token();
 
    /* Return a string representation of the Token's contents. */
    std::string to_string() const;
 
    /* Return the contents of the line where the Token was located. */
    std::string line() const;
 
    /* Return a copy of the original token, but adding the contents of the line where
    the token was located. */
    static Token make_with_line(const Token& origin, const std::string& line_text);
 
    void operator=(const Token& other);
 
    /* Convert the Token into a blank token (does not delete it), so that the Normalizer can remove it afterwards.
    This is to not alter vectors while iterating over them. */
    static void erase(Token& tk);
 
    /* Return a copy of this Token, but with its TokenType changed. */
    Token copy_with_new_type(const TokenType& type);
 
    /* Iterate over two vectors of Tokens, starting from count for vector A, starting from 0 for vector B, checking if
    the current Tokens' types match. If at any point they don't, return false. Else, return true. */
    static bool match_token_types(const std::vector<Token>& a, const std::vector<Token>& b, int count);
 
  private:
    std::string line_text;
};
 
/* typedef to make it easier to see a what a std::vector of tokens is being used for. */
typedef std::vector<Token> TokenStream;