/// <summary>
///
/// </summary>
/// <returns></returns>
public Token ConsumeToken()
{
Token result;
// Consume whitespace before doing anything
while (LookAheadChar == '\n' || LookAheadChar == ' ')
{
ConsumeChar();
}
if (LookAheadChar != '\0') // More text to read
{
result = new Token(CurrentLineNumber, CurrentColumnNumber, CurrentCharacterIndex, "", Token.Types.EOF); // Assume EOF; store current position after consuming whitespace
if (Char.IsLetter(LookAheadChar)) // Identifier or keyword
{
result.Text += ConsumeChar(); // Append
while (Char.IsLetterOrDigit(LookAheadChar) || LookAheadChar == '_') // If next character is a letter, digit, or underscore
result.Text += ConsumeChar(); // Append
result.Text = result.Text.ToLower(); // Case insensitivity
switch (result.Text) // Check if keyword or identifier
{
case "string":
result.Type = Token.Types.STRING;
break;
case "case":
result.Type = Token.Types.CASE;
break;
case "int":
result.Type = Token.Types.INT;
break;
case "for":
result.Type = Token.Types.FOR;
break;
case "bool":
result.Type = Token.Types.BOOL;
break;
case "true":
result.Type = Token.Types.TRUE;
break;
case "false":
result.Type = Token.Types.FALSE;
break;
case "and":
result.Type = Token.Types.AND;
break;
case "float":
result.Type = Token.Types.FLOAT;
break;
case "or":
result.Type = Token.Types.OR;
break;
case "global":
result.Type = Token.Types.GLOBAL;
break;
case "not":
result.Type = Token.Types.NOT;
break;
case "in":
result.Type = Token.Types.IN;
break;
case "out":
result.Type = Token.Types.OUT;
break;
case "procedure":
result.Type = Token.Types.PROCEDURE;
break;
case "then":
result.Type = Token.Types.THEN;
break;
case "return":
result.Type = Token.Types.RETURN;
break;
case "else":
result.Type = Token.Types.ELSE;
break;
case "end":
result.Type = Token.Types.END;
break;
case "program":
result.Type = Token.Types.PROGRAM;
break;
case "is":
result.Type = Token.Types.IS;
break;
case "begin":
result.Type = Token.Types.BEGIN;
break;
case "if":
result.Type = Token.Types.IF;
break;
default:
result.Type = Token.Types.IDENTIFIER;
break;
}
}
else if (Char.IsDigit(LookAheadChar)) // Number (integer or float)
{
result.Text += ConsumeChar(); // Append
while (Char.IsDigit(LookAheadChar)) // If next character is a digit
result.Text += ConsumeChar(); // Append
if (LookAheadChar == '.') // Float
{
result.Text += ConsumeChar();
while (Char.IsDigit(LookAheadChar)) // If next character is a digit
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.FLOAT_VALUE;
}
else // Integer
result.Type = Token.Types.INT_VALUE;
}
else if (LookAheadChar == '"')
{
result.Text += ConsumeChar(); // Append first quote
while (LookAheadChar != '"') // Consume until matching quote
{
if (Char.IsLetterOrDigit(LookAheadChar) || // If valid string char
LookAheadChar == ' ' ||
LookAheadChar == '_' ||
LookAheadChar == ',' ||
LookAheadChar == ';' ||
LookAheadChar == ':' ||
LookAheadChar == '.')
result.Text += ConsumeChar(); // Append
else
{
ErrorHandler.LexerError(this, "Non-string character found.");
}
}
result.Text += ConsumeChar(); // Append matching quote
result.Type = Token.Types.STRING_VALUE;
}
else if (LookAheadChar == '&')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.AND;
}
else if (LookAheadChar == '|')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.OR;
}
else if (LookAheadChar == '+')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.ADD;
}
else if (LookAheadChar == '-')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.SUB;
}
else if (LookAheadChar == '*')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.MUL;
}
else if (LookAheadChar == '/') // Comment or DIV
{
result.Text += ConsumeChar(); // Append
if (LookAheadChar == '/') // Comment
{
while (CurrentCharacterIndex < ProgramText.Length && ConsumeChar() != '\n') ; // Ignore comment
return ConsumeToken(); // Pass along next valid token
}
else
result.Type = Token.Types.DIV;
}
else if (LookAheadChar == ';')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.SEMICOLON;
}
else if (LookAheadChar == ',')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.COMMA;
}
else if (LookAheadChar == '(')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.OPEN_PARENTHESIS;
}
else if (LookAheadChar == ')')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.CLOSE_PARENTHESIS;
}
else if (LookAheadChar == '[')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.OPEN_BRACKET;
}
else if (LookAheadChar == ']')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.CLOSE_BRACKET;
}
else if (LookAheadChar == '{')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.OPEN_BRACE;
}
else if (LookAheadChar == '}')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.CLOSE_BRACE;
}
else if (LookAheadChar == '=')
{
result.Text += ConsumeChar(); // Append
if (LookAheadChar == '=')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.EQUAL;
}
else
ErrorHandler.LexerError(this, "Invalid operator.");
}
else if (LookAheadChar == '<')
{
result.Text += ConsumeChar(); // Append
if (LookAheadChar == '=')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.LESSTHAN_EQUAL;
}
else
result.Type = Token.Types.LESSTHAN;
}
else if (LookAheadChar == '>')
{
result.Text += ConsumeChar(); // Append
if (LookAheadChar == '=')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.GREATERTHAN_EQUAL;
}
else
result.Type = Token.Types.GREATERTHAN;
}
else if (LookAheadChar == '!')
{
result.Text += ConsumeChar(); // Append
if (LookAheadChar == '=')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.NOT_EQUAL;
}
else
ErrorHandler.LexerError(this, "Invalid operator.");
}
else if (LookAheadChar == ':')
{
result.Text += ConsumeChar(); // Append
if (LookAheadChar == '=')
{
result.Text += ConsumeChar(); // Append
result.Type = Token.Types.ASSIGN;
}
else
result.Type = Token.Types.COLON;
}
else
{
ErrorHandler.LexerError(this, "Invalid token.");
}
}
else
{
result = new Token(CurrentLineNumber, CurrentColumnNumber, CurrentCharacterIndex, "", Token.Types.EOF); // Assume EOF
}
Token oldLookahead = LookAheadToken; // Save old lookahead token
lookaheadToken = result; // Set new lookahead token to consumed token
Console.WriteLine(result.Type.ToString() + "\t" + result.Text);
return oldLookahead; // Return the consumed lookahead
}
public Token ConsumeOperatorToken(Token.Types op)
{
if (LookAheadToken.Type == op)
return ConsumeToken();
else
{
ErrorHandler.LexerError(this, "Missing '" + op.ToString() + "' operator.");
return null; // If ExitOnError is disabled
}
}
public static bool IsTypeMark(Token.Types typeMark)
{
if (typeMark == Token.Types.BOOL ||
typeMark == Token.Types.FLOAT ||
typeMark == Token.Types.INT ||
typeMark == Token.Types.STRING)
return true;
else
return false;
}
public Token ConsumeKeywordToken(Token.Types keyword)
{
if (LookAheadToken.Type == keyword)
{
return ConsumeToken();
}
else
{
ErrorHandler.MissingRequiredKeywordError(this, keyword);
return null; // If ExitOnError is disabled
}
}
public static void MissingRequiredKeywordError(Lexer lex, Token.Types keyword)
{
ParserError(lex, "Keyword '" + keyword.ToString() + "' required.");
}
