/// <summary>
/// Lexicalizes a word type
/// </summary>
private void LexicalizeWord()
{
if (IsKeyword(tokenizer.GetTextValue())) // Is probably keyword
{
//Check for symbols directly after keyword
if (IsSymbol(tokenizer.PeekCharacter()))
{ //It is not a keyword, followed directly by symbol, so maybe a path, etc.
LexicalizeIdentifier();
}
else
{ // We are dealing with an keyword
TokenStream.Add(new Token(tokenizer.GetTextValue(), TokenType.KEYWORD, tokenizer.GetScannedLines()));
CurrentToken = tokenizer.NextToken();
}
}
else if (IsIdentifier(tokenizer.GetTextValue()))
{
LexicalizeIdentifier();
return;
}
else
{
throw new StreamTokenizerException("Invalid token: " + CurrentToken, tokenizer.GetScannedLines());
}
}