public void EmbeddingTest()
{
//Set up tokenizer
WaebricLexer lexer = new WaebricLexer(new StringReader("\"pre<\"\\\">\">post\""));
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
//Test token
Assert.AreEqual(1, tokens.GetSize());
Assert.AreEqual(TokenType.EMBEDDING, tokens.Peek(1).GetType());
//Get embedding and test inner tokens
EmbeddingToken parsedToken = (EmbeddingToken) tokens.NextToken();
TokenIterator embeddingTokens = parsedToken.GetTokenIterator();
Assert.AreEqual(7, embeddingTokens.GetSize());
Assert.AreEqual("\"", embeddingTokens.Peek(1).GetValue().ToString());
Assert.AreEqual("pre", embeddingTokens.Peek(2).GetValue().ToString());
Assert.AreEqual("<", embeddingTokens.Peek(3).GetValue().ToString());
Assert.AreEqual("\\\">", embeddingTokens.Peek(4).GetValue().ToString());
Assert.AreEqual(">", embeddingTokens.Peek(5).GetValue().ToString());
Assert.AreEqual("post", embeddingTokens.Peek(6).GetValue().ToString());
Assert.AreEqual("\"", embeddingTokens.Peek(7).GetValue().ToString());
}
public void DoubleFunctionDefinition()
{
//First parse the initial file and then request the rest
//Let's lexicalize the file
StreamReader sourceStream = new StreamReader("../../../../TestChecker/WaebricTestFiles/doublefunctiondefinition.wae");
WaebricLexer lexer = new WaebricLexer(sourceStream);
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
//Lets parse the file
WaebricParser parser = new WaebricParser(tokens);
parser.Parse();
SyntaxTree parsedTree = parser.GetTree();
//Initialize ModuleCache with correct DirectoryPath
ModuleCache.Instance.SetDirectoryPath("../../../../TestChecker/WaebricTestFiles/");
//Lets check the tree
WaebricChecker checker = new WaebricChecker();
List<Exception> checkerExceptions = checker.CheckSyntaxTree(parsedTree);
//Exception function already defined should be in list
Assert.AreEqual(1, checkerExceptions.Count);
Assert.AreEqual(typeof(FunctionAlreadyDefined), checkerExceptions.ToArray()[0].GetType());
}
public void ComplexEmbeddingTest()
{
//Set up tokenizer
WaebricLexer lexer = new WaebricLexer(new StringReader("\"<a(href=\"http://www.microsoft.com\") \"Microsoft Corp\">\""));
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
//Test token
Assert.AreEqual(1, tokens.GetSize());
Assert.AreEqual(TokenType.EMBEDDING, tokens.Peek(1).GetType());
//Test tokens in embedding
EmbeddingToken embeddingToken = (EmbeddingToken) tokens.NextToken();
TokenIterator embeddingTokens = embeddingToken.GetTokenIterator();
Assert.AreEqual(12, embeddingTokens.GetSize());
Assert.AreEqual("\"", embeddingTokens.Peek(1).GetValue().ToString());
Assert.AreEqual("", embeddingTokens.Peek(2).GetValue().ToString());
Assert.AreEqual("<", embeddingTokens.Peek(3).GetValue().ToString());
Assert.AreEqual("a", embeddingTokens.Peek(4).GetValue().ToString());
Assert.AreEqual("(", embeddingTokens.Peek(5).GetValue().ToString());
Assert.AreEqual("href", embeddingTokens.Peek(6).GetValue().ToString());
Assert.AreEqual("=", embeddingTokens.Peek(7).GetValue().ToString());
Assert.AreEqual("http://www.microsoft.com", embeddingTokens.Peek(8).GetValue().ToString());
Assert.AreEqual(")", embeddingTokens.Peek(9).GetValue().ToString());
Assert.AreEqual("Microsoft Corp", embeddingTokens.Peek(10).GetValue().ToString());
Assert.AreEqual(">", embeddingTokens.Peek(11).GetValue().ToString());
Assert.AreEqual("\"", embeddingTokens.Peek(12).GetValue().ToString());
}
private static String Path; //Path of file to compile
#endregion Fields
#region Methods
public static void Main(string[] args)
{
Console.WriteLine("Waebric Compiler/Interpreter v1.0");
Console.WriteLine("---------------------------------");
if (args.Length == 1)
{ //There is one file specified.
Path = args[0];
}
else
{
Console.WriteLine("WeabricCompiler: no input file specified.");
Console.Read(); //Testing purposes only
return;
}
//Let's lexicalize the file
StreamReader sourceStream = new StreamReader(Path);
WaebricLexer lexer = new WaebricLexer(sourceStream);
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
if (tokens.GetSize() == 0)
{ //Not tokens parsed
Console.WriteLine("WaebricCompiler: Empty file or comments only.");
return; //Nothing to compile so end program
}
//Lets parse the file
WaebricParser parser = new WaebricParser(tokens);
parser.Parse();
SyntaxTree parsedTree = parser.GetTree();
//Initialize ModuleCache with correct DirectoryPath
ModuleCache.Instance.SetDirectoryPath(GetDirectoryPath());
//Lets check the tree
WaebricChecker checker = new WaebricChecker();
checker.CheckSyntaxTree(parsedTree);
//Lets interpret the tree and generate XHTML
WaebricInterpreter interpreter = new WaebricInterpreter();
interpreter.InterpretAST(parsedTree);
}
public void ModuleParserComplexModuleNameTest()
{
SyntaxTree tree;
//Create lexer to tokenize stream
WaebricLexer lexer = new WaebricLexer(new StringReader("module test.test2.test3"));
lexer.LexicalizeStream();
//Retrieve tokenIterator from lexer and lets parse it
WaebricParser parser = new WaebricParser(lexer.GetTokenIterator());
parser.Parse();
//Test if root is modulelist and it contains the right module
tree = parser.GetTree();
Module module = tree.GetRoot();
Assert.AreEqual(3, module.GetModuleId().GetIdentifiers().Count);
Assert.AreEqual("test.test2.test3", module.GetModuleId().ToString());
}
public void ModuleParserImportTest()
{
SyntaxTree tree;
//Create lexer to tokenize stream
WaebricLexer lexer = new WaebricLexer(new StringReader("module test\n\nimport importtest"));
lexer.LexicalizeStream();
//Test if stream is lexicalized into 4 tokens
Assert.IsTrue(lexer.GetTokenIterator().GetSize() == 4);
//Retrieve tokenIterator from lexer and lets parse it
WaebricParser parser = new WaebricParser(lexer.GetTokenIterator());
parser.Parse();
//Test tree structure
tree = parser.GetTree();
Module module = tree.GetRoot();
}
/// <summary>
/// Lexicalizes a buffer. When lexicalized it adds the tokens to the given list.
/// </summary>
/// <param name="tokens">TokenList to add new tokens to</param>
/// <param name="buffer">Buffer to lexicalize</param>
/// <param name="line">Linenumber of startposition</param>
private void LexicalizeBuffer(List <Token> tokens, String buffer, int line)
{
if (buffer == null || buffer == "")
{ //no data
return;
}
//Create new lexer and lexicalize buffer
StringReader stringReader = new StringReader(buffer);
WaebricLexer lexer = new WaebricLexer(stringReader);
lexer.SetLine(line);
lexer.LexicalizeStream();
//Add new tokens to list
List <Token> scannedTokens = lexer.GetTokenList();
for (int i = 0; i <= (scannedTokens.Count - 1); i++)
{
tokens.Add(scannedTokens[i]);
}
}
public void WaebricSymbolTokenTest()
{
//Set up tokenizer
WaebricLexer lexer = new WaebricLexer(new StringReader("'test"));
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
//Test token
Assert.AreEqual(1, tokens.GetSize());
Token token = tokens.NextToken();
Assert.AreEqual(TokenType.WAEBRICSYMBOL, token.GetType());
Assert.AreEqual("test", token.GetValue().ToString());
}
public void CleanUp()
{
lexer = null;
}
/// <summary>
/// Initialize test
/// </summary>
/// <param name="stream">Stream to lexicalize</param>
/// <returns>TokenIterator</returns>
private TokenIterator Init(String stream)
{
lexer = new WaebricLexer(new StringReader(stream));
lexer.LexicalizeStream();
return lexer.GetTokenIterator();
}
public void WaebricCheckerImportTest()
{
//First parse the initial file and then request the rest
//Let's lexicalize the file
StreamReader sourceStream = new StreamReader("../../../../TestChecker/WaebricTestFiles/home.wae");
WaebricLexer lexer = new WaebricLexer(sourceStream);
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
//Lets parse the file
WaebricParser parser = new WaebricParser(tokens);
parser.Parse();
SyntaxTree parsedTree = parser.GetTree();
//Initialize ModuleCache with correct DirectoryPath
ModuleCache.Instance.SetDirectoryPath("../../../../TestChecker/WaebricTestFiles/");
//Lets check the tree
WaebricChecker checker = new WaebricChecker();
List<Exception> checkerExceptions = checker.CheckSyntaxTree(parsedTree);
//Test output
Assert.AreEqual(0, checkerExceptions.Count);
//Test if all modules except tree root are in cache
Assert.IsTrue(ModuleCache.Instance.ContainsModule("first"));
Assert.IsTrue(ModuleCache.Instance.ContainsModule("second"));
Assert.IsTrue(ModuleCache.Instance.ContainsModule("common"));
}
public void ModuleParserSingleModuleTest()
{
SyntaxTree tree;
//Create lexer to tokenize stream
WaebricLexer lexer = new WaebricLexer(new StringReader("module test"));
lexer.LexicalizeStream();
//Retrieve tokenIterator from lexer and lets parse it
WaebricParser parser = new WaebricParser(lexer.GetTokenIterator());
parser.Parse();
//Test if root is modulelist and it contains the right module
tree = parser.GetTree();
Module module = tree.GetRoot();
String[] identifiers = module.GetModuleId().GetIdentifiers().ToArray();
Assert.AreEqual(1, identifiers.Length);
Assert.AreEqual("test", identifiers[0]);
}
/// <summary>
/// Request an specified module
/// </summary>
/// <param name="indentifier">ModuleId of requested module</param>
/// <returns>Requested module if available</returns>
public Module RequestModule(ModuleId identifier)
{
if (ModuleTable.ContainsKey(identifier))
{ //Module already loaded so return instance of module
return (Module) ModuleTable[identifier];
}
//Module not cached, so load it
StreamReader moduleStream = new StreamReader(GetPath(identifier));
//Lexicalize and parse it
WaebricLexer lexer = new WaebricLexer(moduleStream);
lexer.LexicalizeStream();
WaebricParser parser = new WaebricParser(lexer.GetTokenIterator());
parser.Parse();
//Get module of tree
SyntaxTree tree = parser.GetTree();
//Add module to hashtable
Module requestedModule = tree.GetRoot();
ModuleTable.Add(identifier, requestedModule);
return requestedModule;
}
public void ModuleParserSiteTest()
{
SyntaxTree tree = new SyntaxTree();
//Create lexer to tokenize stream
WaebricLexer lexer = new WaebricLexer(new StringReader("module test\n\nsite\n site/index.html : home() ; site/index2.html : home()\nend"));
lexer.LexicalizeStream();
//Parse tokenized stream
ModuleParser parser = new ModuleParser(lexer.GetTokenIterator());
tree.SetRoot(parser.ParseModule());
//Check module
Module module = tree.GetRoot();
Assert.IsTrue(module.GetModuleId().ToString() == "test");
Assert.AreEqual(0, module.GetImports().Count); //No imports
Assert.AreEqual(0, module.GetFunctionDefinitions().Count); //No function definitions
Assert.AreEqual(1, module.GetSites().Count); //One site
//Check site
Site site = (Site) module.GetSites().Get(0);
Assert.AreEqual(2, site.GetMappings().Count);
}
/// <summary>
/// Lexicalizes a buffer. When lexicalized it adds the tokens to the given list.
/// </summary>
/// <param name="tokens">TokenList to add new tokens to</param>
/// <param name="buffer">Buffer to lexicalize</param>
/// <param name="line">Linenumber of startposition</param>
private void LexicalizeBuffer(List<Token> tokens, String buffer, int line)
{
if (buffer == null || buffer == "")
{ //no data
return;
}
//Create new lexer and lexicalize buffer
StringReader stringReader = new StringReader(buffer);
WaebricLexer lexer = new WaebricLexer(stringReader);
lexer.SetLine(line);
lexer.LexicalizeStream();
//Add new tokens to list
List<Token> scannedTokens = lexer.GetTokenList();
for (int i = 0; i <= (scannedTokens.Count - 1); i++)
{
tokens.Add(scannedTokens[i]);
}
}
public void TestComplexStream()
{
WaebricLexer lexer = new WaebricLexer(new StringReader("module test\n\nsite site/index.html : home()\nend"));
lexer.LexicalizeStream();
Assert.IsTrue(lexer.GetTokenList().Count == 13);
}
public void SingleQuoteTest()
{
//Set up tokenizer
WaebricLexer lexer = new WaebricLexer(new StringReader("\""));
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
Assert.AreEqual(1, tokens.GetSize());
Assert.AreEqual(TokenType.SYMBOL, tokens.Peek(1).GetType());
Assert.AreEqual("\"", tokens.Peek(1).GetValue().ToString());
}
public void SingleLineCommentTest()
{
//Set up tokenizer
WaebricLexer lexer = new WaebricLexer(new StringReader("//this is a comment"));
lexer.LexicalizeStream();
TokenIterator tokens = lexer.GetTokenIterator();
Assert.AreEqual(0, tokens.GetSize());
}
/// <summary>
/// Lexicalizes a quote
/// </summary>
private void LexicalizeQuote()
{
//Store current line number location for backtracking
int tempLine = tokenizer.GetScannedLines();
//Hold previous char for recognizing escape chars
char previousChar = '\0';
bool IsString = false;
//Skip " token, only text is interesting
CurrentToken = tokenizer.NextToken();
//Ignore comments, due urls, etc
tokenizer.SetIgnoreComments(true);
//Check if this text is comment text
Token[] tempArray = TokenStream.ToArray();
if (tempArray[tempArray.Length - 1].GetType() == TokenType.KEYWORD && tempArray[tempArray.Length - 1].GetValue().ToString() == "comment")
{
IsString = true;
}
//Retrieve possible quoted text
StringBuilder stringBuilder = new StringBuilder();
tokenizer.SetIgnoreNumeric(true);
while (tokenizer.GetCharacterValue() != '\"' || previousChar == '\\') //Scan until non escaped " found
{
if(CurrentToken == StreamTokenizer.EOF)
{ // End of file, so it wasn't a quoted part but just a single "
tokenizer.SetIgnoreComments(false);
tokenizer.SetIgnoreNumeric(false);
//First add a single quote as token
TokenStream.Add(new Token("\"", TokenType.SYMBOL, tempLine));
//Second, scan remaining string
WaebricLexer tempLexer = new WaebricLexer(new StringReader(stringBuilder.ToString()));
tempLexer.LexicalizeStream();
List<Token> tempTokenList = tempLexer.GetTokenList();
//Add all tokens to stream
foreach(Token currentToken in tempTokenList)
{
TokenStream.Add(new Token(currentToken.GetValue(), currentToken.GetType(), (currentToken.GetLine()+tempLine)));
}
return; //Lexicalizing done
}
else if(tokenizer.GetCharacterValue() == '<' && !IsString)
{ //Embedding found, so lexicalize embedding
LexicalizeEmbedding(stringBuilder.ToString());
tokenizer.SetIgnoreComments(false);
tokenizer.SetIgnoreNumeric(false);
return;
}
//Get next part and add it to stringBuilder
stringBuilder.Append(tokenizer.ToString());
previousChar = tokenizer.GetCharacterValue();
CurrentToken = tokenizer.NextToken();
}
tokenizer.SetIgnoreComments(false);
tokenizer.SetIgnoreNumeric(false);
//Check if string is correct quote text
if (IsString)
{
if (!IsCorrectString(stringBuilder.ToString()))
{
throw new StreamTokenizerException("String Text containts non valid characters", tempLine);
}
}
TokenStream.Add(new Token(stringBuilder.ToString(), TokenType.TEXT, tempLine));
//Skip " token, only text is interesting
CurrentToken = tokenizer.NextToken();
}
/// <summary>
/// Lexicalizes a quote
/// </summary>
private void LexicalizeQuote()
{
//Store current line number location for backtracking
int tempLine = tokenizer.GetScannedLines();
//Hold previous char for recognizing escape chars
char previousChar = '\0';
bool IsString = false;
//Skip " token, only text is interesting
CurrentToken = tokenizer.NextToken();
//Ignore comments, due urls, etc
tokenizer.SetIgnoreComments(true);
//Check if this text is comment text
Token[] tempArray = TokenStream.ToArray();
if (tempArray[tempArray.Length - 1].GetType() == TokenType.KEYWORD && tempArray[tempArray.Length - 1].GetValue().ToString() == "comment")
{
IsString = true;
}
//Retrieve possible quoted text
StringBuilder stringBuilder = new StringBuilder();
tokenizer.SetIgnoreNumeric(true);
while (tokenizer.GetCharacterValue() != '\"' || previousChar == '\\') //Scan until non escaped " found
{
if (CurrentToken == StreamTokenizer.EOF)
{ // End of file, so it wasn't a quoted part but just a single "
tokenizer.SetIgnoreComments(false);
tokenizer.SetIgnoreNumeric(false);
//First add a single quote as token
TokenStream.Add(new Token("\"", TokenType.SYMBOL, tempLine));
//Second, scan remaining string
WaebricLexer tempLexer = new WaebricLexer(new StringReader(stringBuilder.ToString()));
tempLexer.LexicalizeStream();
List <Token> tempTokenList = tempLexer.GetTokenList();
//Add all tokens to stream
foreach (Token currentToken in tempTokenList)
{
TokenStream.Add(new Token(currentToken.GetValue(), currentToken.GetType(), (currentToken.GetLine() + tempLine)));
}
return; //Lexicalizing done
}
else if (tokenizer.GetCharacterValue() == '<' && !IsString)
{ //Embedding found, so lexicalize embedding
LexicalizeEmbedding(stringBuilder.ToString());
tokenizer.SetIgnoreComments(false);
tokenizer.SetIgnoreNumeric(false);
return;
}
//Get next part and add it to stringBuilder
stringBuilder.Append(tokenizer.ToString());
previousChar = tokenizer.GetCharacterValue();
CurrentToken = tokenizer.NextToken();
}
tokenizer.SetIgnoreComments(false);
tokenizer.SetIgnoreNumeric(false);
//Check if string is correct quote text
if (IsString)
{
if (!IsCorrectString(stringBuilder.ToString()))
{
throw new StreamTokenizerException("String Text containts non valid characters", tempLine);
}
}
TokenStream.Add(new Token(stringBuilder.ToString(), TokenType.TEXT, tempLine));
//Skip " token, only text is interesting
CurrentToken = tokenizer.NextToken();
}
