/// <summary>
/// Find the next token
/// </summary>
/// <returns>The type of the next token</returns>
/// <remarks>Sets tokenSpelling to be the characters in the token</remarks>
private TokenType ScanToken()
{
TokenSpelling.Clear();
if (Reader.Current == default(char))
{
// Read the end of the file
TakeIt();
return(TokenType.EndOfText);
}
//tokens for single chars
else if (Reader.Current == ';')
{
TakeIt();
return(TokenType.Semicolon);
}
else if (Reader.Current == ':')
{
TakeIt();
//checks for Becomes
if (Reader.Current == '=')
{
TakeIt();
return(TokenType.Becomes);
}
else
{
TakeIt();
return(TokenType.Colon);
}
}
else if (Reader.Current == '(')
{
TakeIt();
return(TokenType.LeftBracket);
}
else if (Reader.Current == ')')
{
TakeIt();
return(TokenType.RightBracket);
}
else if (Reader.Current == '~')
{
TakeIt();
return(TokenType.Is);
}
//tokens for operators
if (IsOperator(Reader.Current))
{
TakeIt();
return(TokenType.Operator);
}
//tokens for character literals
if (Reader.Current == '\'')
{
TakeIt();
TakeIt();
if (Reader.Current == '\'')
{
TakeIt();
return(TokenType.CharLiteral);
}
return(TokenType.Error);
}
//tokens for strings
else if (char.IsLetter(Reader.Current))
{
TakeIt();
while (char.IsLetter(Reader.Current))
{
TakeIt();
}
if (TokenTypes.IsKeyword(TokenSpelling))
{
return(TokenTypes.GetTokenForKeyword(TokenSpelling));
}
else
{
return(TokenType.Identifier);
}
}
//tokens for integers
else if (char.IsDigit(Reader.Current))
{
TakeIt();
while (char.IsDigit(Reader.Current))
{
TakeIt();
}
return(TokenType.IntLiteral);
}
else
{
// Encountered a character we weren't expecting
TakeIt();
return(TokenType.Error);
}
}
/// <summary>
/// Find the next token
/// </summary>
/// <returns>The type of the next token</returns>
/// <remarks>Sets tokenSpelling to be the characters in the token</remarks>
private TokenType ScanToken()
{
TokenSpelling.Clear();
if (char.IsLetter(Reader.Current) || Reader.Current == '_')
{
while (Reader.Current == '_')
{
TakeIt();
}
// Reading an identifier
//TakeIt();
while (char.IsLetterOrDigit(Reader.Current))
{
TakeIt();
}
if (TokenTypes.IsKeyword(TokenSpelling))
{
return(TokenTypes.GetTokenForKeyword(TokenSpelling));
}
else
{
return(TokenType.Identifier);
}
}
else if (Reader.Current == '0' || char.IsDigit(Reader.Current))
{
// Reading an integer
TakeIt();
while (char.IsDigit(Reader.Current))
{
TakeIt();
}
return(TokenType.IntLiteral);
}
else if (Reader.Current == '’')
{
// Read a char literal
TakeIt();
if (Graphic(Reader.Current))
{
TakeIt();
TakeIt(); // take the end of the quote;
}
return(TokenType.CharLiteral);
}
else if (IsOperator(Reader.Current))
{
// Read an operator
TakeIt();
return(TokenType.Operator);
}
else if (Reader.Current == '!')
{
TakeIt();
return(TokenType.Operator);
}
else if (Reader.Current == ':')
{
// Read an :
// Is it a : or a :=
TakeIt();
if (Reader.Current == '=')
{
TakeIt();
return(TokenType.Becomes);
}
else
{
return(TokenType.Colon);
}
}
else if (Reader.Current == ';')
{
// Read a ;
TakeIt();
return(TokenType.Semicolon);
}
else if (Reader.Current == '~')
{
// Read a ~
TakeIt();
return(TokenType.Is);
}
else if (Reader.Current == '(')
{
// Read a (
TakeIt();
return(TokenType.LeftBracket);
}
else if (Reader.Current == ')')
{
// Read a )
TakeIt();
return(TokenType.RightBracket);
}
else if (Reader.Current == '\'')
{
// Read a '
TakeIt();
// Take whatever the character is
TakeIt();
// Try getting the closing '
if (Reader.Current == '\'')
{
TakeIt();
return(TokenType.CharLiteral);
}
else
{
// Could do some better error handling here but we weren't asked to
return(TokenType.Error);
}
}
else if (Reader.Current == '/')
{
TakeIt();
return(TokenType.Operator);
}
else if (Reader.Current == default(char))
{
// Read the end of the file
TakeIt();
return(TokenType.EndOfText);
}
else
{
// Encountered a character we weren't expecting
TakeIt();
return(TokenType.Error);
}
}
/// <summary>
/// Find the next token
/// </summary>
/// <returns>The type of the next token</returns>
/// <remarks>Sets tokenSpelling to be the characters in the token</remarks>
private TokenType ScanToken()
{
TokenSpelling.Clear();
if (Reader.Current == '_' || char.IsLetter(Reader.Current))
{
TakeIt();
while (Reader.Current == '_')
{
TakeIt();
}
if (char.IsLetter(Reader.Current))
{
TakeIt();
while (char.IsLetterOrDigit(Reader.Current))
{
TakeIt();
}
}
if (TokenTypes.IsKeyword(TokenSpelling))
{
return(TokenTypes.GetTokenForKeyword(TokenSpelling));
}
else
{
return(TokenType.Identifier);
}
}
//integer
else if (char.IsDigit(Reader.Current))
{
TakeIt();
while (char.IsDigit(Reader.Current))
{
TakeIt();
}
return(TokenType.IntLiteral);
}
else if (IsGraphic(Reader.Current) || char.IsLetterOrDigit(Reader.Current))
{
TakeIt();
return(TokenType.Graphic);
}
else if (Reader.Current == '\'')
{
//Read '
TakeIt();
//Read the character
TakeIt();
while (char.IsLetterOrDigit(Reader.Current) || IsGraphic(Reader.Current))
{
TakeIt();
}
if (Reader.Current == '\'')
{
TakeIt();
return(TokenType.CharLiteral);
}
else
{
return(TokenType.Error);
}
}
//operator
else if (IsOperator(Reader.Current))
{
//Read operator
TakeIt();
return(TokenType.Operator);
}
else if (Reader.Current == ':')
{
//Read ":"
//Check for :=
TakeIt();
if (Reader.Current == '=')
{
TakeIt();
return(TokenType.Becomes);
}
else
{
return(TokenType.Colon);
}
}
else if (Reader.Current == ';')
{
//Read ;
TakeIt();
return(TokenType.Semicolon);
}
else if (Reader.Current == '~')
{
//Read ~
TakeIt();
return(TokenType.Is);
}
else if (Reader.Current == '(')
{
//Read (
TakeIt();
return(TokenType.LeftBracket);
}
else if (Reader.Current == ')')
{
//Read )
TakeIt();
return(TokenType.RightBracket);
}
if (Reader.Current == default(char))
{
// Read the end of the file
TakeIt();
return(TokenType.EndOfText);
}
else
{
// Encountered a character we weren't expecting
TakeIt();
return(TokenType.Error);
}
}
/// <summary>
/// Find the next token
/// </summary>
/// <returns></returns>
private TokenType ScanToken()
{
TokenSpelling.Clear();
if (char.IsLetter(Reader.Current) || Reader.Current == '_')
{
// Reading an identifier
TakeIt();
while (Reader.Current == '_' || char.IsLetter(Reader.Current))
{
TakeIt();
}
while (char.IsLetterOrDigit(Reader.Current))
{
TakeIt();
}
if (TokenTypes.IsKeyword(TokenSpelling))
{
return(TokenTypes.GetTokenForKeyword(TokenSpelling));
}
else
{
return(TokenType.Identifier);
}
}
else if (char.IsDigit(Reader.Current))
{
// Reading an integer
if (Reader.Current == '0')
{
TakeIt();
}
else if (Reader.Current > '0') // non zero digit
{
while (char.IsDigit(Reader.Current)) // digit
{
TakeIt();
}
}
return(TokenType.IntLiteral);
}
else if (IsOperator(Reader.Current))
{
// Read an operator
TakeIt();
return(TokenType.Operator);
}
else if (Reader.Current == ':')
{
// Read an :
// Is it a : or a :=
TakeIt();
if (Reader.Current == '=')
{
TakeIt();
return(TokenType.Becomes);
}
else
{
return(TokenType.Colon);
}
}
else if (Reader.Current == ';')
{
TakeIt();
return(TokenType.Semicolon);
}
else if (Reader.Current == '~')
{
TakeIt();
return(TokenType.Is);
}
else if (Reader.Current == '(')
{
TakeIt();
return(TokenType.LeftBracket);
}
else if (Reader.Current == ')')
{
TakeIt();
return(TokenType.RightBracket);
}
else if (Reader.Current == '\'')
{
// Read a '
TakeIt();
// Scan <graphic>
if (char.IsLetter(Reader.Current))
{
TakeIt();
}
else if (char.IsDigit(Reader.Current))
{
TakeIt();
}
else if (Reader.Current == '.')
{
TakeIt();
}
else if (Reader.Current == '?')
{
TakeIt();
}
else if (char.IsWhiteSpace(Reader.Current))
{
TakeIt();
}
// Try getting the closing '
if (Reader.Current == '\'')
{
TakeIt();
return(TokenType.CharLiteral);
}
else
{
return(TokenType.UnacceptableSeqError);
}
}
else if (IsOperator(Reader.Current))
{
TakeIt();
return(TokenType.Operator);
}
else if (Reader.Current == default(char))
{
TakeIt();
return(TokenType.EndOfText);
}
else
{
TakeIt();
return(TokenType.UnknownInputError);
}
}
/// <summary>
/// Find the next token
/// </summary>
/// <returns>The type of the next token</returns>
/// <remarks>Sets tokenSpelling to be the characters in the token</remarks>
private TokenType ScanToken()
{
TokenSpelling.Clear();
if (char.IsLetter(Reader.Current))
{
// Reading an identifier
TakeIt(); // takes the charcter
while (isLowerLetterDigit(Reader.Current))
{
TakeIt(); //keep taking it as long as its letter or a digit
}
if (TokenTypes.IsKeyword(TokenSpelling)) // if it is a keyword...
{
return(TokenTypes.GetTokenForKeyword(TokenSpelling)); // return word
}
else if (TokenSpelling.ToString().Any(char.IsUpper))
{
return(TokenType.Error);
}
else
{
return(TokenType.Identifier); // if not, retun idenitfer
}
}
else if (Reader.Current == '_')
{
TakeIt(); // take the _
while (isLowerLetter(Reader.Current))
{ // if the next bit is a leter keep taking it
TakeIt();
}
// once it finds a non letter item...
return(TokenType.Identifier);
}
else if (Reader.Current == '’')
{
TakeIt();
// do while is lower letter or a figit or a punctuation mark or white space
while (isLowerLetterDigit(Reader.Current) || IsPunctuation(Reader.Current) || IsWhiteSpace(Reader.Current)) // == graphic
{
TakeIt();
}
if (Reader.Current == '’') //and ends with ' it is a character literal
{
return(TokenType.CharLiteral); //it is a character literal
}
else
{
return(TokenType.Error);
}
}
else if (char.IsDigit(Reader.Current))
{
// Reading an integer
TakeIt();
while (char.IsDigit(Reader.Current))
{
TakeIt();
}
return(TokenType.IntLiteral);
}
else if (IsOperator(Reader.Current))
{
// Read an operator
TakeIt();
// if the next this is an = sign then take it to before delaring it as an oeprator
if (Reader.Current == '=')
{
TakeIt();
}
return(TokenType.Operator);
}
else if (Reader.Current == ':')
{
// Read an :
// Is it a : or a :=
TakeIt();
if (Reader.Current == '=')
{
TakeIt();
return(TokenType.Becomes);
}
else
{
return(TokenType.Colon);
}
}
else if (Reader.Current == ';')
{
// Read a ;
TakeIt();
return(TokenType.Semicolon);
}
else if (Reader.Current == '~')
{
// Read a ~
TakeIt();
return(TokenType.Is);
}
else if (Reader.Current == '(')
{
// Read a (
TakeIt();
return(TokenType.LeftBracket);
}
else if (Reader.Current == ')')
{
// Read a )
TakeIt();
return(TokenType.RightBracket);
}
else if (Reader.Current == '\'')
{
// Read a '
TakeIt();
// Take whatever the character is
TakeIt();
// Try getting the closing '
if (Reader.Current == '\'')
{
TakeIt();
return(TokenType.CharLiteral);
}
else
{
// Could do some better error handling here but we weren't asked to
return(TokenType.Error);
}
}
else if (Reader.Current == default(char))
{
// Read the end of the file
TakeIt();
return(TokenType.EndOfText);
}
else
{
// Encountered a character we weren't expecting
TakeIt();
return(TokenType.Error);
}
}
/// <summary>
/// Find the next token
/// </summary>
/// <returns>The type of the next token</returns>
/// <remarks>Sets tokenSpelling to be the characters in the token</remarks>
private TokenType ScanToken()
{
TokenSpelling.Clear();
if (char.IsLetter(Reader.Current))
{
// Identifiers and Keywords
TakeIt();
while (char.IsLetter(Reader.Current))
{
TakeIt();
}
if (TokenTypes.IsKeyword(TokenSpelling))
{
return(TokenTypes.GetTokenForKeyword(TokenSpelling));
}
else
{
return(TokenType.Identifier);
}
}
else if (char.IsDigit(Reader.Current))
{
// Integer Literals
TakeIt();
while (char.IsDigit(Reader.Current))
{
TakeIt();
}
return(TokenType.IntLiteral);
}
else if (Reader.Current == ':')
{
// Handle ":" and ":="
TakeIt();
if (Reader.Current == '=')
{
TakeIt();
return(TokenType.Becomes);
}
return(TokenType.Colon);
}
else if (Reader.Current == ';')
{
// ";"
TakeIt();
return(TokenType.Semicolon);
}
else if (Reader.Current == '(')
{
// "("
TakeIt();
return(TokenType.LeftBracket);
}
else if (Reader.Current == ')')
{
// ")"
TakeIt();
return(TokenType.RightBracket);
}
else if (Reader.Current == '~')
{
// "Is"
TakeIt();
return(TokenType.Is);
}
else if (Reader.Current == '\'')
{
// Charcter Literal
TakeIt();
TakeIt();
if (Reader.Current == '\'')
{
TakeIt();
return(TokenType.CharLiteral);
}
return(TokenType.Error);
}
else if (IsOperator(Reader.Current))
{
// Operators
TakeIt();
return(TokenType.Operator);
}
else if (Reader.Current == default(char))
{
// Read the end of the file
TakeIt();
return(TokenType.EndOfText);
}
else
{
// Encountered a character we weren't expecting
TakeIt();
return(TokenType.Error);
}
}
