// Test if each token generated from a test file generates tokens with correct names // A newline is inserted after each scope public void GenerateTokensCorrectly() { // Initialise Lexer Lexer l = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "/tang.tokens.json")); // Read from test file IEnumerable <Token> tokens = l.Analyse(@"int16 a = 1 bool on = true while(on == true) a = a + 1"); /* * Assert that we have 26 tokens in testSourceFile.tang * int16 identifier = numeral * newline * bool identifier = true * newline * while ( identifier == true ) * indent identifier = identifier + numeral newline * dedent newline eof */ Assert.Equal(tokens.Count(), 26); // For safety measures, test at random places that the token is of correct name Assert.Equal(tokens.ElementAt(0).Name, "int16"); Assert.Equal(tokens.ElementAt(4).Name, "newline"); Assert.Equal(tokens.ElementAt(10).Name, "while"); Assert.Equal(tokens.ElementAt(13).Name, "=="); Assert.Equal(tokens.ElementAt(16).Name, "indent"); Assert.Equal(tokens.ElementAt(25).Name, "eof"); }
public List <Token> Process(Lexer lexer, string path, IEnumerable <Token> tokens) { bool onlyImportsYet = true; List <Token> newList = new List <Token>(); foreach (Token token in tokens) { if (onlyImportsYet && token.Name == "import" && !imports.Contains(token.Value)) { var importedTokens = lexer.Analyse(File.ReadAllText(path + token.Value.Replace("import ", "") + ".tang"), token.Value.Replace("import ", "") + ".tang"); foreach (var importedToken in importedTokens) { importedToken.FileName = token.Value.Replace("import ", "") + ".tang"; } var newTokens = Process(lexer, path, importedTokens); if (newTokens == null) { throw new Exception("Import not found"); } foreach (var newToken in newTokens) { if (newToken.Name != "eof") { newList.Add(newToken); } else { newList.RemoveAt(newList.Count - 1); // To remove last inserted newline } } imports.Add(token.Value); } else if (token.Name == "newline") { // Yield but dont set onlyImportsYet newList.Add(token); } else if (onlyImportsYet == false && token.Name == "import") { throw new Exception("Imports must be at start of file"); } else if (token.Name == "import" && imports.Contains(token.Value)) { // Do exactly nothing. We already imported this } else { onlyImportsYet = false; newList.Add(token); } } return(newList); }
// In cases where it is unsure which token to produce, read until match ends and produce a token from that public void SplitTokensCorrectly() { // Initialise Lexer Lexer l = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "/tang.tokens.json")); IEnumerable <Token> tokens = l.Analyse("=== 2test"); Assert.Equal(tokens.ElementAt(0).Name, "=="); Assert.Equal(tokens.ElementAt(1).Name, "="); Assert.Equal(tokens.ElementAt(2).Name, "numeral"); Assert.Equal(tokens.ElementAt(3).Name, "identifier"); // All programs have a newline and eof token inserted at the end Assert.Equal(tokens.ElementAt(4).Name, "newline"); Assert.Equal(tokens.ElementAt(5).Name, "eof"); }
public void AddExpressionSpecialTestLexer() { // Initialise Lexer Lexer l = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "/tang.tokens.json")); // Read from test file IEnumerable <Token> tokens = l.Analyse(File.ReadAllText(AppContext.BaseDirectory + "/Testfiles/tang/AddExpression.tang")); /* * int8 a = 1 + 2 + 3 * should be * int8 identifier = numeral + numeral + numeral newline eof */ Assert.Equal(tokens.Count(), 10); Assert.Equal(tokens.ElementAt(2).Name, "="); Assert.Equal(tokens.ElementAt(4).Name, "+"); Assert.Equal(tokens.ElementAt(7).Name, "numeral"); Assert.Equal(tokens.ElementAt(8).Name, "newline"); Assert.Equal(tokens.ElementAt(9).Name, "eof"); }
// Test if attributes are given correctly, e.g. singleline and row/column public void GenereteCorrectTokenAttributes() { // Initialise Lexer Lexer le = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "/tang.tokens.json")); // Read from another file, tokens should be SimpleType Identifier Assign Number eof (int16 a = 1) IEnumerable <Token> tokens = le.Analyse(@"int16 a = 1 bool on = true while(on == true) a = a + 1"); // '=' should be at row 0 and column 8 since there are 8 symbols until '=' is hit Assert.Equal(tokens.ElementAt(2).Row, 0); Assert.Equal(tokens.ElementAt(2).Column, 8); // 'on' should be at row 1 and column 5 Assert.Equal(tokens.ElementAt(6).Row, 1); Assert.Equal(tokens.ElementAt(6).Column, 5); }
public void AddExpressionSpecialTestAST() { // Source file to produce an AST from string source = System.IO.File.ReadAllText(AppContext.BaseDirectory + "/Testfiles/tang/AddExpression.tang"); // File path relative to where the debug file is located which is in a land far, far away Lexer l = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "../../../../../../docs/tang.tokens.json")); // Call the Analyse method from the Lexical Analysis class var tokens = l.Analyse(source); // Initialise a program parser ProgramParser parser = new ProgramParser(); // Convert tokens to Parsing.Data.Token var tokenEnumerator = tokens.Select(t => new Parsing.Data.Token() { Name = t.Name, Value = t.Value }).GetEnumerator(); tokenEnumerator.MoveNext(); // Produce a parse tree by calling the ParseProgram method var parseTree = parser.ParseProgram(tokenEnumerator); // Create a new instance of the ProgramToASTTranslator class var astTranslator = new Translation.ProgramToAST.ProgramToASTTranslator(); // Use the ProgramToASTTranslator with parseTree as parameter to get the AST AST.Data.AST ast = astTranslator.TranslatetoAST(parseTree) as AST.Data.AST; // Below is a hardcoded tree of how the AST is expected to look var astExpected = new AST.Data.AST(true) { new AST.Data.GlobalStatement(true) { new AST.Data.Statement(true) { new AST.Data.IntegerDeclarationInit(true) { new AST.Data.IntType(true) { new AST.Data.Token() { Name = "int8" } }, new AST.Data.Token() { Name = "identifier" }, new AST.Data.Token() { Name = "=" }, new AST.Data.IntegerExpression(true) { new AST.Data.AddExpression(true) { new AST.Data.IntegerExpression(true) { new AST.Data.AddExpression(true) { new AST.Data.IntegerExpression(true) { new AST.Data.Token() { Name = "numeral" } }, new AST.Data.Token() { Name = "+" }, new AST.Data.IntegerExpression(true) { new AST.Data.Token() { Name = "numeral" } } } }, new AST.Data.Token() { Name = "+" }, new AST.Data.IntegerExpression(true) { new AST.Data.Token() { Name = "numeral" } } } } } } }, new AST.Data.Token() { Name = "eof" } }; // Call a method to walk the tree's nodes and test if their names are equal TreeAsserter(ast, astExpected); }
public void ConvertToASTCorrectly() { /* Alias.tang file: * int8 a * a = 2 * int8 b * b = a * tokens: * int8 identifier newline * identifier = numeral newline * int8 identifier newline * identifier = identifier newline eof */ // When running the method ParseProgram on an enumerator of these tokens, the result is a big parse tree // The goal of the ProgramToASTTranslator is to take this parse tree as input and output an AST // The AST output is written by hand from the .tang file, and then it is asserted if the output is equivalent to the expected output string source = System.IO.File.ReadAllText(AppContext.BaseDirectory + "/Testfiles/tang/Alias.tang"); // File path relative to where the debug file is located which is in a land far, far away Lexer l = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "../../../../../../docs/tang.tokens.json")); var tokens = l.Analyse(source); ProgramParser parser = new ProgramParser(); var tokenEnumerator = tokens.Select(t => new Parsing.Data.Token() { Name = t.Name, Value = t.Value }).GetEnumerator(); tokenEnumerator.MoveNext(); var parseTree = parser.ParseProgram(tokenEnumerator); // Create a new instance of the ProgramToASTTranslator class var astTranslator = new Translation.ProgramToAST.ProgramToASTTranslator(); // Use the ProgramToASTTranslator with parseTree as parameter to get the AST AST.Data.AST ast = astTranslator.TranslatetoAST(parseTree) as AST.Data.AST; // Below is a hardcoded tree of how the AST is expected to look var astExpected = new AST.Data.AST(true) { new AST.Data.GlobalStatement(true) { new AST.Data.CompoundGlobalStatement(true) { new AST.Data.GlobalStatement(true) { new AST.Data.Statement(true) { new AST.Data.IntegerDeclaration(true) { new AST.Data.IntType(true) { new AST.Data.Token() { Name = "int8" } }, new AST.Data.Token() { Name = "identifier" } } } }, new AST.Data.Token() { Name = "newline" }, new AST.Data.GlobalStatement(true) { new AST.Data.CompoundGlobalStatement(true) { new AST.Data.GlobalStatement(true) { new AST.Data.Statement(true) { new AST.Data.IntegerAssignment(true) { new AST.Data.Token() { Name = "identifier" }, new AST.Data.Token() { Name = "=" }, new AST.Data.IntegerExpression(true) { new AST.Data.Token() { Name = "numeral" } } } } }, new AST.Data.Token() { Name = "newline" }, new AST.Data.GlobalStatement(true) { new AST.Data.CompoundGlobalStatement(true) { new AST.Data.GlobalStatement(true) { new AST.Data.Statement(true) { new AST.Data.IntegerDeclaration(true) { new AST.Data.IntType(true) { new AST.Data.Token() { Name = "int8" } }, new AST.Data.Token() { Name = "identifier" } } } }, new AST.Data.Token() { Name = "newline" }, new AST.Data.GlobalStatement(true) { new AST.Data.Statement(true) { new AST.Data.IntegerAssignment(true) { new AST.Data.Token() { Name = "identifier" }, new AST.Data.Token() { Name = "=" }, new AST.Data.IntegerExpression(true) { new AST.Data.Token() { Name = "identifier" } } } } } } } } } }, }, new AST.Data.Token() { Name = "eof" } }; // Call a method to walk the tree's nodes and test if their names are equal TreeAsserter(ast, astExpected); }
public void AddExpressionSpecialTestCodeGenerator() { Lexer lexer = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "/../../../../../docs/tang.tokens.json")); var tokens = lexer.Analyse("int8 a = 1 + 2 + 3"); var tokenEnumerator = tokens.Select(t => new Parsing.Data.Token() { Name = t.Name, Value = t.Value }).GetEnumerator(); tokenEnumerator.MoveNext(); Preprocessor preprocessor = new Preprocessor(); tokens = preprocessor.Process(lexer, null, tokens); ProgramParser parser = new ProgramParser(); var parseTree = parser.ParseProgram(tokenEnumerator); var astTranslator = new Translation.ProgramToAST.ProgramToASTTranslator(); AST.Data.AST ast = astTranslator.TranslatetoAST(parseTree) as AST.Data.AST; var cTranslator = new Translation.ASTToC.ASTToCTranslator(); C.Data.C c = cTranslator.Translate(ast) as C.Data.C; // c is the autogenerated tree, we have to manually create one and assert their equality var cExpected = new C.Data.C(true) { new C.Data.Declaration(true) { new C.Data.CompoundDeclaration(true) { new C.Data.Declaration(true) { new C.Data.CompoundDeclaration(true) { new C.Data.Declaration(true) { new C.Data.Token() { Name = "EPSILON" } }, new C.Data.Declaration(true) { new C.Data.IntegerDeclaration(true) { new C.Data.IntType(true) { new C.Data.Token() { Name = "signed" }, new C.Data.Token() { Name = "char" } }, new C.Data.Token() { Name = "identifier" } }, new C.Data.Token() { Name = ";" } } } }, new C.Data.Declaration(true) { new C.Data.CompoundDeclaration(true) { new C.Data.Declaration(true) { new C.Data.FunctionPrototype(true) { new C.Data.Token() { Name = "signed" }, new C.Data.Token() { Name = "long" }, new C.Data.Token() { Name = "Pow" }, new C.Data.Token() { Name = "(" }, new C.Data.Token() { Name = "signed" }, new C.Data.Token() { Name = "long" }, new C.Data.Token() { Name = "a" }, new C.Data.Token() { Name = "," }, new C.Data.Token() { Name = "signed" }, new C.Data.Token() { Name = "long" }, new C.Data.Token() { Name = "b" }, new C.Data.Token() { Name = ")" } }, new C.Data.Token() { Name = ";" } }, new C.Data.Declaration(true) { new C.Data.FunctionPrototype(true) { new C.Data.Type(true) { new C.Data.Token() { Name = "void" } }, new C.Data.Token() { Name = "main" }, new C.Data.Token() { Name = "(" }, new C.Data.Token() { Name = ")" } }, new C.Data.Token() { Name = ";" } } } } } }, new C.Data.Function(true) { new C.Data.CompoundFunction(true) { new C.Data.Function(true) { new C.Data.CompoundFunction(true) { new C.Data.Function(true) { new C.Data.Token() { Name = "signed" }, new C.Data.Token() { Name = "long" }, new C.Data.Token() { Name = "Pow" }, new C.Data.Token() { Name = "(" }, new C.Data.Token() { Name = "signed" }, new C.Data.Token() { Name = "long" }, new C.Data.Token() { Name = "a" }, new C.Data.Token() { Name = "," }, new C.Data.Token() { Name = "signed" }, new C.Data.Token() { Name = "long" }, new C.Data.Token() { Name = "b" }, new C.Data.Token() { Name = ")" }, new C.Data.Token() { Name = "{" }, new C.Data.Token() { Name = "signed" }, new C.Data.Token() { Name = "long" }, new C.Data.Token() { Name = "r" }, new C.Data.Token() { Name = "=" }, new C.Data.Token() { Name = "1" }, new C.Data.Token() { Name = "," }, new C.Data.Token() { Name = "i" }, new C.Data.Token() { Name = ";" }, new C.Data.Token() { Name = "for" }, new C.Data.Token() { Name = "(" }, new C.Data.Token() { Name = "i" }, new C.Data.Token() { Name = "=" }, new C.Data.Token() { Name = "0" }, new C.Data.Token() { Name = ";" }, new C.Data.Token() { Name = "i" }, new C.Data.Token() { Name = "<" }, new C.Data.Token() { Name = "b" }, new C.Data.Token() { Name = ";" }, new C.Data.Token() { Name = "i" }, new C.Data.Token() { Name = "++" }, new C.Data.Token() { Name = ")" }, new C.Data.Token() { Name = "{" }, new C.Data.Token() { Name = "r" }, new C.Data.Token() { Name = "*=" }, new C.Data.Token() { Name = "a" }, new C.Data.Token() { Name = ";" }, new C.Data.Token() { Name = "}" }, new C.Data.Token() { Name = "return" }, new C.Data.Token() { Name = "r" }, new C.Data.Token() { Name = ";" }, new C.Data.Token() { Name = "}" } }, new C.Data.Function(true) { new C.Data.Token() { Name = "EPSILON" } } } }, new C.Data.Function(true) { new C.Data.Type(true) { new C.Data.Token() { Name = "void" } }, new C.Data.Token() { Name = "main" }, new C.Data.Token() { Name = "(" }, new C.Data.Token() { Name = ")" }, new C.Data.Token() { Name = "{" }, new C.Data.Declaration(true) { new C.Data.Token() { Name = "EPSILON" } }, new C.Data.Statement(true) { new C.Data.CompoundStatement(true) { new C.Data.Statement(true) { new C.Data.Token() { Name = "EPSILON" } }, new C.Data.Statement(true) { new C.Data.IntegerAssignment(true) { new C.Data.Token() { Name = "identifier" }, new C.Data.Token() { Name = "=" }, new C.Data.IntegerExpression(true) { new C.Data.AddExpression(true) { new C.Data.Token() { Name = "(" }, new C.Data.IntegerExpression(true) { new C.Data.AddExpression(true) { new C.Data.Token() { Name = "(" }, new C.Data.IntegerExpression(true) { new C.Data.Token() { Name = "numeral" } }, new C.Data.Token() { Name = "+" }, new C.Data.IntegerExpression(true) { new C.Data.Token() { Name = "numeral" } }, new C.Data.Token() { Name = ")" } } }, new C.Data.Token() { Name = "+" }, new C.Data.IntegerExpression(true) { new C.Data.Token() { Name = "numeral" } }, new C.Data.Token() { Name = ")" } } } }, new C.Data.Token() { Name = ";" } } } }, new C.Data.Token() { Name = "}" } } } } }; // Use this to print the actual tree //var CCC = c.Accept(new C.Visitors.TreePrintVisitor()); //foreach (var line in CCC) //{ // Debug.WriteLine(line); //} // Call a method to walk the tree's nodes and test if their names are equal TreeAsserter(c, cExpected); }