108 lines
2.6 KiB
C++
108 lines
2.6 KiB
C++
#pragma once
|
|
#include "Location.h"
|
|
#include "sapphirepch.h"
|
|
|
|
/* All current token types. Will change in the future. */
|
|
enum TokenType
|
|
{
|
|
TT_Identifier,
|
|
TT_Number,
|
|
TT_Float,
|
|
TT_Keyword,
|
|
TT_String,
|
|
TT_Plus,
|
|
TT_Minus,
|
|
TT_Mul,
|
|
TT_Div,
|
|
TT_At,
|
|
TT_Equal,
|
|
TT_LessThan,
|
|
TT_GreaterThan,
|
|
TT_LParen,
|
|
TT_RParen,
|
|
TT_LBracket,
|
|
TT_RBracket,
|
|
TT_Semicolon,
|
|
TT_LoadedString,
|
|
TT_EOF,
|
|
TT_Null,
|
|
TT_Equals,
|
|
TT_GTE,
|
|
TT_LTE,
|
|
TT_Period,
|
|
TT_Comma,
|
|
TT_Path,
|
|
TT_Exclamation,
|
|
TT_NEqual,
|
|
TT_LSQB,
|
|
TT_RSQB,
|
|
TT_Type,
|
|
TT_Import,
|
|
TT_Syscall0,
|
|
TT_Syscall1,
|
|
TT_Syscall2,
|
|
TT_Syscall3,
|
|
TT_Syscall4,
|
|
TT_Syscall5,
|
|
TT_CompilerMacro,
|
|
TT_Let,
|
|
TT_In
|
|
};
|
|
|
|
extern const std::string token_strings[];
|
|
|
|
/* Struct to represent tokens generated by the Lexer. */
|
|
struct Token
|
|
{
|
|
TokenType tk_type;
|
|
|
|
int int_value;
|
|
std::string string_value;
|
|
float float_value;
|
|
|
|
Location loc;
|
|
|
|
Token(const TokenType& type);
|
|
|
|
Token(const TokenType& type, const Location& location);
|
|
|
|
Token(const TokenType& type, const int& val, const Location& location);
|
|
|
|
Token(const TokenType& type, const std::string& val, const Location& location);
|
|
|
|
Token(const TokenType& type, const std::string& val);
|
|
|
|
Token(const TokenType& type, const float& val, const Location& location);
|
|
|
|
~Token();
|
|
|
|
/* Return a string representation of the Token's contents. */
|
|
std::string to_string() const;
|
|
|
|
/* Return the contents of the line where the Token was located. */
|
|
std::string line() const;
|
|
|
|
/* Return a copy of the original token, but adding the contents of the line where
|
|
the token was located. */
|
|
static Token make_with_line(const Token& origin, const std::string& line_text);
|
|
|
|
void operator=(const Token& other);
|
|
|
|
/* Convert the Token into a blank token (does not delete it), so that the Normalizer can remove it afterwards.
|
|
This is to not alter vectors while iterating over them. */
|
|
static void erase(Token& tk);
|
|
|
|
/* Return a copy of this Token, but with its TokenType changed. */
|
|
Token copy_with_new_type(const TokenType& type);
|
|
|
|
/* Iterate over two vectors of Tokens, starting from count for vector A, starting from 0 for vector B, checking if
|
|
the current Tokens' types match. If at any point they don't, return false. Else, return true. */
|
|
static bool match_token_types(const std::vector<Token>& a, const std::vector<Token>& b, int count);
|
|
|
|
private:
|
|
std::string line_text;
|
|
};
|
|
|
|
/* typedef to make it easier to see a what a std::vector of tokens is being used for. */
|
|
typedef std::vector<Token> TokenStream;
|