/// <summary>
/// Lexicalizes a word type
/// </summary>
private void LexicalizeWord()
{
if (IsKeyword(tokenizer.GetTextValue())) // Is probably keyword
{
//Check for symbols directly after keyword
if (IsSymbol(tokenizer.PeekCharacter()))
{ //It is not a keyword, followed directly by symbol, so maybe a path, etc.
LexicalizeIdentifier();
}
else
{ // We are dealing with an keyword
TokenStream.Add(new Token(tokenizer.GetTextValue(), TokenType.KEYWORD, tokenizer.GetScannedLines()));
CurrentToken = tokenizer.NextToken();
}
}
else if (IsIdentifier(tokenizer.GetTextValue()))
{
LexicalizeIdentifier();
return;
}
else
{
throw new StreamTokenizerException("Invalid token: " + CurrentToken, tokenizer.GetScannedLines());
}
}
public void TestWaebricInput()
{
//Set up tokenizer and input for tokenizer
Tokenizer = new StreamTokenizer(new StringReader("module test\n\nsite site/index.html : home()\nend"));
//Tokenize stream and do some test with it
int current = Tokenizer.NextToken();
int position = 1;
while (current != StreamTokenizer.EOF)
{
if (current == StreamTokenizer.LAYOUT)
{
current = Tokenizer.NextToken();
continue; //ignore layout
}
switch (position)
{
case 1: //module
Assert.AreEqual("module", Tokenizer.GetTextValue());
break;
case 2: //test
Assert.AreEqual("test", Tokenizer.GetTextValue());
break;
case 3: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 4: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 5: // /
Assert.AreEqual('/', Tokenizer.GetCharacterValue());
break;
case 6: //index
Assert.AreEqual("index", Tokenizer.GetTextValue());
break;
case 7: //.
Assert.AreEqual('.', Tokenizer.GetCharacterValue());
break;
case 8: //html
Assert.AreEqual("html", Tokenizer.GetTextValue());
break;
case 9: //:
Assert.AreEqual(':', Tokenizer.GetCharacterValue());
break;
case 10: //home
Assert.AreEqual("home", Tokenizer.GetTextValue());
break;
case 11: //(
Assert.AreEqual('(', Tokenizer.GetCharacterValue());
break;
case 12: //)
Assert.AreEqual(')', Tokenizer.GetCharacterValue());
break;
case 13: //end
Assert.AreEqual("end", Tokenizer.GetTextValue());
break;
}
position++;
current = Tokenizer.NextToken();
}
}
public void TestWaebricInput()
{
//Set up tokenizer and input for tokenizer
Tokenizer = new StreamTokenizer(new StringReader("module test\n\nsite site/index.html : home()\nend"));
//Tokenize stream and do some test with it
int current = Tokenizer.NextToken();
int position = 1;
while(current != StreamTokenizer.EOF)
{
if (current == StreamTokenizer.LAYOUT)
{
current = Tokenizer.NextToken();
continue; //ignore layout
}
switch (position)
{
case 1: //module
Assert.AreEqual("module", Tokenizer.GetTextValue());
break;
case 2: //test
Assert.AreEqual("test", Tokenizer.GetTextValue());
break;
case 3: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 4: //site
Assert.AreEqual("site", Tokenizer.GetTextValue());
break;
case 5: // /
Assert.AreEqual('/', Tokenizer.GetCharacterValue());
break;
case 6: //index
Assert.AreEqual("index", Tokenizer.GetTextValue());
break;
case 7: //.
Assert.AreEqual('.', Tokenizer.GetCharacterValue());
break;
case 8: //html
Assert.AreEqual("html", Tokenizer.GetTextValue());
break;
case 9: //:
Assert.AreEqual(':', Tokenizer.GetCharacterValue());
break;
case 10: //home
Assert.AreEqual("home", Tokenizer.GetTextValue());
break;
case 11: //(
Assert.AreEqual('(', Tokenizer.GetCharacterValue());
break;
case 12: //)
Assert.AreEqual(')', Tokenizer.GetCharacterValue());
break;
case 13: //end
Assert.AreEqual("end", Tokenizer.GetTextValue());
break;
}
position++;
current = Tokenizer.NextToken();
}
}
