/// <summary> /// Lexicalizes the stream to tokens /// </summary> public void LexicalizeStream() { TokenStream.Clear(); //Clean stream before inserting items CurrentToken = tokenizer.NextToken(); while (CurrentToken != StreamTokenizer.EOF) { switch (CurrentToken) { case StreamTokenizer.LAYOUT: // ignore layout CurrentToken = tokenizer.NextToken(); break; case StreamTokenizer.COMMENT: // ignore comments CurrentToken = tokenizer.NextToken(); break; case StreamTokenizer.WORD: // check word to determine type LexicalizeWord(); break; case StreamTokenizer.NUMBER: // numeric value LexicalizeNumber(); break; case StreamTokenizer.CHARACTER: // Character if (tokenizer.GetCharacterValue() == '"') // Possible a quote { LexicalizeQuote(); } else if (tokenizer.GetCharacterValue() == '\'') //Waebric Symbol ('symbol ) { LexicalizeSymbol(); } else { // Just an character LexicalizeCharacter(); } break; default: //Other tokens are not correct throw new StreamTokenizerException("Invalid token: " + CurrentToken, tokenizer.GetScannedLines()); } } }
public void TestWaebricInput() { //Set up tokenizer and input for tokenizer Tokenizer = new StreamTokenizer(new StringReader("module test\n\nsite site/index.html : home()\nend")); //Tokenize stream and do some test with it int current = Tokenizer.NextToken(); int position = 1; while (current != StreamTokenizer.EOF) { if (current == StreamTokenizer.LAYOUT) { current = Tokenizer.NextToken(); continue; //ignore layout } switch (position) { case 1: //module Assert.AreEqual("module", Tokenizer.GetTextValue()); break; case 2: //test Assert.AreEqual("test", Tokenizer.GetTextValue()); break; case 3: //site Assert.AreEqual("site", Tokenizer.GetTextValue()); break; case 4: //site Assert.AreEqual("site", Tokenizer.GetTextValue()); break; case 5: // / Assert.AreEqual('/', Tokenizer.GetCharacterValue()); break; case 6: //index Assert.AreEqual("index", Tokenizer.GetTextValue()); break; case 7: //. Assert.AreEqual('.', Tokenizer.GetCharacterValue()); break; case 8: //html Assert.AreEqual("html", Tokenizer.GetTextValue()); break; case 9: //: Assert.AreEqual(':', Tokenizer.GetCharacterValue()); break; case 10: //home Assert.AreEqual("home", Tokenizer.GetTextValue()); break; case 11: //( Assert.AreEqual('(', Tokenizer.GetCharacterValue()); break; case 12: //) Assert.AreEqual(')', Tokenizer.GetCharacterValue()); break; case 13: //end Assert.AreEqual("end", Tokenizer.GetTextValue()); break; } position++; current = Tokenizer.NextToken(); } }
public void TestWaebricInput() { //Set up tokenizer and input for tokenizer Tokenizer = new StreamTokenizer(new StringReader("module test\n\nsite site/index.html : home()\nend")); //Tokenize stream and do some test with it int current = Tokenizer.NextToken(); int position = 1; while(current != StreamTokenizer.EOF) { if (current == StreamTokenizer.LAYOUT) { current = Tokenizer.NextToken(); continue; //ignore layout } switch (position) { case 1: //module Assert.AreEqual("module", Tokenizer.GetTextValue()); break; case 2: //test Assert.AreEqual("test", Tokenizer.GetTextValue()); break; case 3: //site Assert.AreEqual("site", Tokenizer.GetTextValue()); break; case 4: //site Assert.AreEqual("site", Tokenizer.GetTextValue()); break; case 5: // / Assert.AreEqual('/', Tokenizer.GetCharacterValue()); break; case 6: //index Assert.AreEqual("index", Tokenizer.GetTextValue()); break; case 7: //. Assert.AreEqual('.', Tokenizer.GetCharacterValue()); break; case 8: //html Assert.AreEqual("html", Tokenizer.GetTextValue()); break; case 9: //: Assert.AreEqual(':', Tokenizer.GetCharacterValue()); break; case 10: //home Assert.AreEqual("home", Tokenizer.GetTextValue()); break; case 11: //( Assert.AreEqual('(', Tokenizer.GetCharacterValue()); break; case 12: //) Assert.AreEqual(')', Tokenizer.GetCharacterValue()); break; case 13: //end Assert.AreEqual("end", Tokenizer.GetTextValue()); break; } position++; current = Tokenizer.NextToken(); } }