/// <summary> /// Lexicalizes a word type /// </summary> private void LexicalizeWord() { if (IsKeyword(tokenizer.GetTextValue())) // Is probably keyword { //Check for symbols directly after keyword if (IsSymbol(tokenizer.PeekCharacter())) { //It is not a keyword, followed directly by symbol, so maybe a path, etc. LexicalizeIdentifier(); } else { // We are dealing with an keyword TokenStream.Add(new Token(tokenizer.GetTextValue(), TokenType.KEYWORD, tokenizer.GetScannedLines())); CurrentToken = tokenizer.NextToken(); } } else if (IsIdentifier(tokenizer.GetTextValue())) { LexicalizeIdentifier(); return; } else { throw new StreamTokenizerException("Invalid token: " + CurrentToken, tokenizer.GetScannedLines()); } }
public void TestWaebricInput() { //Set up tokenizer and input for tokenizer Tokenizer = new StreamTokenizer(new StringReader("module test\n\nsite site/index.html : home()\nend")); //Tokenize stream and do some test with it int current = Tokenizer.NextToken(); int position = 1; while (current != StreamTokenizer.EOF) { if (current == StreamTokenizer.LAYOUT) { current = Tokenizer.NextToken(); continue; //ignore layout } switch (position) { case 1: //module Assert.AreEqual("module", Tokenizer.GetTextValue()); break; case 2: //test Assert.AreEqual("test", Tokenizer.GetTextValue()); break; case 3: //site Assert.AreEqual("site", Tokenizer.GetTextValue()); break; case 4: //site Assert.AreEqual("site", Tokenizer.GetTextValue()); break; case 5: // / Assert.AreEqual('/', Tokenizer.GetCharacterValue()); break; case 6: //index Assert.AreEqual("index", Tokenizer.GetTextValue()); break; case 7: //. Assert.AreEqual('.', Tokenizer.GetCharacterValue()); break; case 8: //html Assert.AreEqual("html", Tokenizer.GetTextValue()); break; case 9: //: Assert.AreEqual(':', Tokenizer.GetCharacterValue()); break; case 10: //home Assert.AreEqual("home", Tokenizer.GetTextValue()); break; case 11: //( Assert.AreEqual('(', Tokenizer.GetCharacterValue()); break; case 12: //) Assert.AreEqual(')', Tokenizer.GetCharacterValue()); break; case 13: //end Assert.AreEqual("end", Tokenizer.GetTextValue()); break; } position++; current = Tokenizer.NextToken(); } }
public void TestWaebricInput() { //Set up tokenizer and input for tokenizer Tokenizer = new StreamTokenizer(new StringReader("module test\n\nsite site/index.html : home()\nend")); //Tokenize stream and do some test with it int current = Tokenizer.NextToken(); int position = 1; while(current != StreamTokenizer.EOF) { if (current == StreamTokenizer.LAYOUT) { current = Tokenizer.NextToken(); continue; //ignore layout } switch (position) { case 1: //module Assert.AreEqual("module", Tokenizer.GetTextValue()); break; case 2: //test Assert.AreEqual("test", Tokenizer.GetTextValue()); break; case 3: //site Assert.AreEqual("site", Tokenizer.GetTextValue()); break; case 4: //site Assert.AreEqual("site", Tokenizer.GetTextValue()); break; case 5: // / Assert.AreEqual('/', Tokenizer.GetCharacterValue()); break; case 6: //index Assert.AreEqual("index", Tokenizer.GetTextValue()); break; case 7: //. Assert.AreEqual('.', Tokenizer.GetCharacterValue()); break; case 8: //html Assert.AreEqual("html", Tokenizer.GetTextValue()); break; case 9: //: Assert.AreEqual(':', Tokenizer.GetCharacterValue()); break; case 10: //home Assert.AreEqual("home", Tokenizer.GetTextValue()); break; case 11: //( Assert.AreEqual('(', Tokenizer.GetCharacterValue()); break; case 12: //) Assert.AreEqual(')', Tokenizer.GetCharacterValue()); break; case 13: //end Assert.AreEqual("end", Tokenizer.GetTextValue()); break; } position++; current = Tokenizer.NextToken(); } }