/// <summary>
/// Lexicalizes the stream to tokens
/// </summary>
public void LexicalizeStream()
{
TokenStream.Clear(); //Clean stream before inserting items
CurrentToken = tokenizer.NextToken();
while (CurrentToken != StreamTokenizer.EOF)
{
switch (CurrentToken)
{
case StreamTokenizer.LAYOUT: // ignore layout
CurrentToken = tokenizer.NextToken();
break;
case StreamTokenizer.COMMENT: // ignore comments
CurrentToken = tokenizer.NextToken();
break;
case StreamTokenizer.WORD: // check word to determine type
LexicalizeWord();
break;
case StreamTokenizer.NUMBER: // numeric value
LexicalizeNumber();
break;
case StreamTokenizer.CHARACTER: // Character
if (tokenizer.GetCharacterValue() == '"') // Possible a quote
{
LexicalizeQuote();
}
else if (tokenizer.GetCharacterValue() == '\'') //Waebric Symbol ('symbol )
{
LexicalizeSymbol();
}
else
{ // Just an character
LexicalizeCharacter();
}
break;
default: //Other tokens are not correct
throw new StreamTokenizerException("Invalid token: " + CurrentToken, tokenizer.GetScannedLines());
}
}
}
public void TestWaebricInput()
{
//Set up tokenizer and input for tokenizer
Tokenizer = new StreamTokenizer(new StringReader("module test\n\nsite site/index.html : home()\nend"));
//Tokenize stream and do some test with it
int current = Tokenizer.NextToken();
int position = 1;
while (current != StreamTokenizer.EOF)
{
if (current == StreamTokenizer.LAYOUT)
{
current = Tokenizer.NextToken();
continue; //ignore layout
}
switch (position)
{
case 1: //module
Assert.AreEqual("module", Tokenizer.GetTextValue());
break;
case 2: //test
Assert.AreEqual("test", Tokenizer.GetTextValue());
break;
case 3: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 4: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 5: // /
Assert.AreEqual('/', Tokenizer.GetCharacterValue());
break;
case 6: //index
Assert.AreEqual("index", Tokenizer.GetTextValue());
break;
case 7: //.
Assert.AreEqual('.', Tokenizer.GetCharacterValue());
break;
case 8: //html
Assert.AreEqual("html", Tokenizer.GetTextValue());
break;
case 9: //:
Assert.AreEqual(':', Tokenizer.GetCharacterValue());
break;
case 10: //home
Assert.AreEqual("home", Tokenizer.GetTextValue());
break;
case 11: //(
Assert.AreEqual('(', Tokenizer.GetCharacterValue());
break;
case 12: //)
Assert.AreEqual(')', Tokenizer.GetCharacterValue());
break;
case 13: //end
Assert.AreEqual("end", Tokenizer.GetTextValue());
break;
}
position++;
current = Tokenizer.NextToken();
}
}
public void TestWaebricInput()
{
//Set up tokenizer and input for tokenizer
Tokenizer = new StreamTokenizer(new StringReader("module test\n\nsite site/index.html : home()\nend"));
//Tokenize stream and do some test with it
int current = Tokenizer.NextToken();
int position = 1;
while(current != StreamTokenizer.EOF)
{
if (current == StreamTokenizer.LAYOUT)
{
current = Tokenizer.NextToken();
continue; //ignore layout
}
switch (position)
{
case 1: //module
Assert.AreEqual("module", Tokenizer.GetTextValue());
break;
case 2: //test
Assert.AreEqual("test", Tokenizer.GetTextValue());
break;
case 3: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 4: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 5: // /
Assert.AreEqual('/', Tokenizer.GetCharacterValue());
break;
case 6: //index
Assert.AreEqual("index", Tokenizer.GetTextValue());
break;
case 7: //.
Assert.AreEqual('.', Tokenizer.GetCharacterValue());
break;
case 8: //html
Assert.AreEqual("html", Tokenizer.GetTextValue());
break;
case 9: //:
Assert.AreEqual(':', Tokenizer.GetCharacterValue());
break;
case 10: //home
Assert.AreEqual("home", Tokenizer.GetTextValue());
break;
case 11: //(
Assert.AreEqual('(', Tokenizer.GetCharacterValue());
break;
case 12: //)
Assert.AreEqual(')', Tokenizer.GetCharacterValue());
break;
case 13: //end
Assert.AreEqual("end", Tokenizer.GetTextValue());
break;
}
position++;
current = Tokenizer.NextToken();
}
}
