/// <summary>
/// Lexicalizes the stream to tokens
/// </summary>
public void LexicalizeStream()
{
TokenStream.Clear(); //Clean stream before inserting items
CurrentToken = tokenizer.NextToken();
while (CurrentToken != StreamTokenizer.EOF)
{
switch (CurrentToken)
{
case StreamTokenizer.LAYOUT: // ignore layout
CurrentToken = tokenizer.NextToken();
break;
case StreamTokenizer.COMMENT: // ignore comments
CurrentToken = tokenizer.NextToken();
break;
case StreamTokenizer.WORD: // check word to determine type
LexicalizeWord();
break;
case StreamTokenizer.NUMBER: // numeric value
LexicalizeNumber();
break;
case StreamTokenizer.CHARACTER: // Character
if (tokenizer.GetCharacterValue() == '"') // Possible a quote
{
LexicalizeQuote();
}
else if (tokenizer.GetCharacterValue() == '\'') //Waebric Symbol ('symbol )
{
LexicalizeSymbol();
}
else
{ // Just an character
LexicalizeCharacter();
}
break;
default: //Other tokens are not correct
throw new StreamTokenizerException("Invalid token: " + CurrentToken, tokenizer.GetScannedLines());
}
}
}
