// Test if each token generated from a test file generates tokens with correct names
// A newline is inserted after each scope
public void GenerateTokensCorrectly()
{
// Initialise Lexer
Lexer l = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "/tang.tokens.json"));
// Read from test file
IEnumerable <Token> tokens = l.Analyse(@"int16 a = 1
bool on = true
while(on == true)
a = a + 1");
/*
* Assert that we have 26 tokens in testSourceFile.tang
* int16 identifier = numeral
* newline
* bool identifier = true
* newline
* while ( identifier == true )
* indent identifier = identifier + numeral newline
* dedent newline eof
*/
Assert.Equal(tokens.Count(), 26);
// For safety measures, test at random places that the token is of correct name
Assert.Equal(tokens.ElementAt(0).Name, "int16");
Assert.Equal(tokens.ElementAt(4).Name, "newline");
Assert.Equal(tokens.ElementAt(10).Name, "while");
Assert.Equal(tokens.ElementAt(13).Name, "==");
Assert.Equal(tokens.ElementAt(16).Name, "indent");
Assert.Equal(tokens.ElementAt(25).Name, "eof");
}
public List <Token> Process(Lexer lexer, string path, IEnumerable <Token> tokens)
{
bool onlyImportsYet = true;
List <Token> newList = new List <Token>();
foreach (Token token in tokens)
{
if (onlyImportsYet && token.Name == "import" && !imports.Contains(token.Value))
{
var importedTokens = lexer.Analyse(File.ReadAllText(path + token.Value.Replace("import ", "") + ".tang"), token.Value.Replace("import ", "") + ".tang");
foreach (var importedToken in importedTokens)
{
importedToken.FileName = token.Value.Replace("import ", "") + ".tang";
}
var newTokens = Process(lexer, path, importedTokens);
if (newTokens == null)
{
throw new Exception("Import not found");
}
foreach (var newToken in newTokens)
{
if (newToken.Name != "eof")
{
newList.Add(newToken);
}
else
{
newList.RemoveAt(newList.Count - 1); // To remove last inserted newline
}
}
imports.Add(token.Value);
}
else if (token.Name == "newline")
{
// Yield but dont set onlyImportsYet
newList.Add(token);
}
else if (onlyImportsYet == false && token.Name == "import")
{
throw new Exception("Imports must be at start of file");
}
else if (token.Name == "import" && imports.Contains(token.Value))
{
// Do exactly nothing. We already imported this
}
else
{
onlyImportsYet = false;
newList.Add(token);
}
}
return(newList);
}
// In cases where it is unsure which token to produce, read until match ends and produce a token from that
public void SplitTokensCorrectly()
{
// Initialise Lexer
Lexer l = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "/tang.tokens.json"));
IEnumerable <Token> tokens = l.Analyse("=== 2test");
Assert.Equal(tokens.ElementAt(0).Name, "==");
Assert.Equal(tokens.ElementAt(1).Name, "=");
Assert.Equal(tokens.ElementAt(2).Name, "numeral");
Assert.Equal(tokens.ElementAt(3).Name, "identifier");
// All programs have a newline and eof token inserted at the end
Assert.Equal(tokens.ElementAt(4).Name, "newline");
Assert.Equal(tokens.ElementAt(5).Name, "eof");
}
public void AddExpressionSpecialTestLexer()
{
// Initialise Lexer
Lexer l = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "/tang.tokens.json"));
// Read from test file
IEnumerable <Token> tokens = l.Analyse(File.ReadAllText(AppContext.BaseDirectory + "/Testfiles/tang/AddExpression.tang"));
/*
* int8 a = 1 + 2 + 3
* should be
* int8 identifier = numeral + numeral + numeral newline eof
*/
Assert.Equal(tokens.Count(), 10);
Assert.Equal(tokens.ElementAt(2).Name, "=");
Assert.Equal(tokens.ElementAt(4).Name, "+");
Assert.Equal(tokens.ElementAt(7).Name, "numeral");
Assert.Equal(tokens.ElementAt(8).Name, "newline");
Assert.Equal(tokens.ElementAt(9).Name, "eof");
}
// Test if attributes are given correctly, e.g. singleline and row/column
public void GenereteCorrectTokenAttributes()
{
// Initialise Lexer
Lexer le = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "/tang.tokens.json"));
// Read from another file, tokens should be SimpleType Identifier Assign Number eof (int16 a = 1)
IEnumerable <Token> tokens = le.Analyse(@"int16 a = 1
bool on = true
while(on == true)
a = a + 1");
// '=' should be at row 0 and column 8 since there are 8 symbols until '=' is hit
Assert.Equal(tokens.ElementAt(2).Row, 0);
Assert.Equal(tokens.ElementAt(2).Column, 8);
// 'on' should be at row 1 and column 5
Assert.Equal(tokens.ElementAt(6).Row, 1);
Assert.Equal(tokens.ElementAt(6).Column, 5);
}
public void AddExpressionSpecialTestAST()
{
// Source file to produce an AST from
string source = System.IO.File.ReadAllText(AppContext.BaseDirectory + "/Testfiles/tang/AddExpression.tang");
// File path relative to where the debug file is located which is in a land far, far away
Lexer l = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "../../../../../../docs/tang.tokens.json"));
// Call the Analyse method from the Lexical Analysis class
var tokens = l.Analyse(source);
// Initialise a program parser
ProgramParser parser = new ProgramParser();
// Convert tokens to Parsing.Data.Token
var tokenEnumerator = tokens.Select(t => new Parsing.Data.Token()
{
Name = t.Name, Value = t.Value
}).GetEnumerator();
tokenEnumerator.MoveNext();
// Produce a parse tree by calling the ParseProgram method
var parseTree = parser.ParseProgram(tokenEnumerator);
// Create a new instance of the ProgramToASTTranslator class
var astTranslator = new Translation.ProgramToAST.ProgramToASTTranslator();
// Use the ProgramToASTTranslator with parseTree as parameter to get the AST
AST.Data.AST ast = astTranslator.TranslatetoAST(parseTree) as AST.Data.AST;
// Below is a hardcoded tree of how the AST is expected to look
var astExpected = new AST.Data.AST(true)
{
new AST.Data.GlobalStatement(true)
{
new AST.Data.Statement(true)
{
new AST.Data.IntegerDeclarationInit(true)
{
new AST.Data.IntType(true)
{
new AST.Data.Token()
{
Name = "int8"
}
},
new AST.Data.Token()
{
Name = "identifier"
},
new AST.Data.Token()
{
Name = "="
},
new AST.Data.IntegerExpression(true)
{
new AST.Data.AddExpression(true)
{
new AST.Data.IntegerExpression(true)
{
new AST.Data.AddExpression(true)
{
new AST.Data.IntegerExpression(true)
{
new AST.Data.Token()
{
Name = "numeral"
}
},
new AST.Data.Token()
{
Name = "+"
},
new AST.Data.IntegerExpression(true)
{
new AST.Data.Token()
{
Name = "numeral"
}
}
}
},
new AST.Data.Token()
{
Name = "+"
},
new AST.Data.IntegerExpression(true)
{
new AST.Data.Token()
{
Name = "numeral"
}
}
}
}
}
}
},
new AST.Data.Token()
{
Name = "eof"
}
};
// Call a method to walk the tree's nodes and test if their names are equal
TreeAsserter(ast, astExpected);
}
public void ConvertToASTCorrectly()
{
/* Alias.tang file:
* int8 a
* a = 2
* int8 b
* b = a
* tokens:
* int8 identifier newline
* identifier = numeral newline
* int8 identifier newline
* identifier = identifier newline eof
*/
// When running the method ParseProgram on an enumerator of these tokens, the result is a big parse tree
// The goal of the ProgramToASTTranslator is to take this parse tree as input and output an AST
// The AST output is written by hand from the .tang file, and then it is asserted if the output is equivalent to the expected output
string source = System.IO.File.ReadAllText(AppContext.BaseDirectory + "/Testfiles/tang/Alias.tang");
// File path relative to where the debug file is located which is in a land far, far away
Lexer l = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "../../../../../../docs/tang.tokens.json"));
var tokens = l.Analyse(source);
ProgramParser parser = new ProgramParser();
var tokenEnumerator = tokens.Select(t => new Parsing.Data.Token()
{
Name = t.Name, Value = t.Value
}).GetEnumerator();
tokenEnumerator.MoveNext();
var parseTree = parser.ParseProgram(tokenEnumerator);
// Create a new instance of the ProgramToASTTranslator class
var astTranslator = new Translation.ProgramToAST.ProgramToASTTranslator();
// Use the ProgramToASTTranslator with parseTree as parameter to get the AST
AST.Data.AST ast = astTranslator.TranslatetoAST(parseTree) as AST.Data.AST;
// Below is a hardcoded tree of how the AST is expected to look
var astExpected = new AST.Data.AST(true)
{
new AST.Data.GlobalStatement(true)
{
new AST.Data.CompoundGlobalStatement(true)
{
new AST.Data.GlobalStatement(true)
{
new AST.Data.Statement(true)
{
new AST.Data.IntegerDeclaration(true)
{
new AST.Data.IntType(true)
{
new AST.Data.Token()
{
Name = "int8"
}
},
new AST.Data.Token()
{
Name = "identifier"
}
}
}
},
new AST.Data.Token()
{
Name = "newline"
},
new AST.Data.GlobalStatement(true)
{
new AST.Data.CompoundGlobalStatement(true)
{
new AST.Data.GlobalStatement(true)
{
new AST.Data.Statement(true)
{
new AST.Data.IntegerAssignment(true)
{
new AST.Data.Token()
{
Name = "identifier"
},
new AST.Data.Token()
{
Name = "="
},
new AST.Data.IntegerExpression(true)
{
new AST.Data.Token()
{
Name = "numeral"
}
}
}
}
},
new AST.Data.Token()
{
Name = "newline"
},
new AST.Data.GlobalStatement(true)
{
new AST.Data.CompoundGlobalStatement(true)
{
new AST.Data.GlobalStatement(true)
{
new AST.Data.Statement(true)
{
new AST.Data.IntegerDeclaration(true)
{
new AST.Data.IntType(true)
{
new AST.Data.Token()
{
Name = "int8"
}
},
new AST.Data.Token()
{
Name = "identifier"
}
}
}
},
new AST.Data.Token()
{
Name = "newline"
},
new AST.Data.GlobalStatement(true)
{
new AST.Data.Statement(true)
{
new AST.Data.IntegerAssignment(true)
{
new AST.Data.Token()
{
Name = "identifier"
},
new AST.Data.Token()
{
Name = "="
},
new AST.Data.IntegerExpression(true)
{
new AST.Data.Token()
{
Name = "identifier"
}
}
}
}
}
}
}
}
}
},
},
new AST.Data.Token()
{
Name = "eof"
}
};
// Call a method to walk the tree's nodes and test if their names are equal
TreeAsserter(ast, astExpected);
}
public void AddExpressionSpecialTestCodeGenerator()
{
Lexer lexer = new Lexer(File.ReadAllText(AppContext.BaseDirectory + "/../../../../../docs/tang.tokens.json"));
var tokens = lexer.Analyse("int8 a = 1 + 2 + 3");
var tokenEnumerator = tokens.Select(t => new Parsing.Data.Token()
{
Name = t.Name, Value = t.Value
}).GetEnumerator();
tokenEnumerator.MoveNext();
Preprocessor preprocessor = new Preprocessor();
tokens = preprocessor.Process(lexer, null, tokens);
ProgramParser parser = new ProgramParser();
var parseTree = parser.ParseProgram(tokenEnumerator);
var astTranslator = new Translation.ProgramToAST.ProgramToASTTranslator();
AST.Data.AST ast = astTranslator.TranslatetoAST(parseTree) as AST.Data.AST;
var cTranslator = new Translation.ASTToC.ASTToCTranslator();
C.Data.C c = cTranslator.Translate(ast) as C.Data.C;
// c is the autogenerated tree, we have to manually create one and assert their equality
var cExpected = new C.Data.C(true)
{
new C.Data.Declaration(true)
{
new C.Data.CompoundDeclaration(true)
{
new C.Data.Declaration(true)
{
new C.Data.CompoundDeclaration(true)
{
new C.Data.Declaration(true)
{
new C.Data.Token()
{
Name = "EPSILON"
}
},
new C.Data.Declaration(true)
{
new C.Data.IntegerDeclaration(true)
{
new C.Data.IntType(true)
{
new C.Data.Token()
{
Name = "signed"
},
new C.Data.Token()
{
Name = "char"
}
},
new C.Data.Token()
{
Name = "identifier"
}
},
new C.Data.Token()
{
Name = ";"
}
}
}
},
new C.Data.Declaration(true)
{
new C.Data.CompoundDeclaration(true)
{
new C.Data.Declaration(true)
{
new C.Data.FunctionPrototype(true)
{
new C.Data.Token()
{
Name = "signed"
},
new C.Data.Token()
{
Name = "long"
},
new C.Data.Token()
{
Name = "Pow"
},
new C.Data.Token()
{
Name = "("
},
new C.Data.Token()
{
Name = "signed"
},
new C.Data.Token()
{
Name = "long"
},
new C.Data.Token()
{
Name = "a"
},
new C.Data.Token()
{
Name = ","
},
new C.Data.Token()
{
Name = "signed"
},
new C.Data.Token()
{
Name = "long"
},
new C.Data.Token()
{
Name = "b"
},
new C.Data.Token()
{
Name = ")"
}
},
new C.Data.Token()
{
Name = ";"
}
},
new C.Data.Declaration(true)
{
new C.Data.FunctionPrototype(true)
{
new C.Data.Type(true)
{
new C.Data.Token()
{
Name = "void"
}
},
new C.Data.Token()
{
Name = "main"
},
new C.Data.Token()
{
Name = "("
},
new C.Data.Token()
{
Name = ")"
}
},
new C.Data.Token()
{
Name = ";"
}
}
}
}
}
},
new C.Data.Function(true)
{
new C.Data.CompoundFunction(true)
{
new C.Data.Function(true)
{
new C.Data.CompoundFunction(true)
{
new C.Data.Function(true)
{
new C.Data.Token()
{
Name = "signed"
},
new C.Data.Token()
{
Name = "long"
},
new C.Data.Token()
{
Name = "Pow"
},
new C.Data.Token()
{
Name = "("
},
new C.Data.Token()
{
Name = "signed"
},
new C.Data.Token()
{
Name = "long"
},
new C.Data.Token()
{
Name = "a"
},
new C.Data.Token()
{
Name = ","
},
new C.Data.Token()
{
Name = "signed"
},
new C.Data.Token()
{
Name = "long"
},
new C.Data.Token()
{
Name = "b"
},
new C.Data.Token()
{
Name = ")"
},
new C.Data.Token()
{
Name = "{"
},
new C.Data.Token()
{
Name = "signed"
},
new C.Data.Token()
{
Name = "long"
},
new C.Data.Token()
{
Name = "r"
},
new C.Data.Token()
{
Name = "="
},
new C.Data.Token()
{
Name = "1"
},
new C.Data.Token()
{
Name = ","
},
new C.Data.Token()
{
Name = "i"
},
new C.Data.Token()
{
Name = ";"
},
new C.Data.Token()
{
Name = "for"
},
new C.Data.Token()
{
Name = "("
},
new C.Data.Token()
{
Name = "i"
},
new C.Data.Token()
{
Name = "="
},
new C.Data.Token()
{
Name = "0"
},
new C.Data.Token()
{
Name = ";"
},
new C.Data.Token()
{
Name = "i"
},
new C.Data.Token()
{
Name = "<"
},
new C.Data.Token()
{
Name = "b"
},
new C.Data.Token()
{
Name = ";"
},
new C.Data.Token()
{
Name = "i"
},
new C.Data.Token()
{
Name = "++"
},
new C.Data.Token()
{
Name = ")"
},
new C.Data.Token()
{
Name = "{"
},
new C.Data.Token()
{
Name = "r"
},
new C.Data.Token()
{
Name = "*="
},
new C.Data.Token()
{
Name = "a"
},
new C.Data.Token()
{
Name = ";"
},
new C.Data.Token()
{
Name = "}"
},
new C.Data.Token()
{
Name = "return"
},
new C.Data.Token()
{
Name = "r"
},
new C.Data.Token()
{
Name = ";"
},
new C.Data.Token()
{
Name = "}"
}
},
new C.Data.Function(true)
{
new C.Data.Token()
{
Name = "EPSILON"
}
}
}
},
new C.Data.Function(true)
{
new C.Data.Type(true)
{
new C.Data.Token()
{
Name = "void"
}
},
new C.Data.Token()
{
Name = "main"
},
new C.Data.Token()
{
Name = "("
},
new C.Data.Token()
{
Name = ")"
},
new C.Data.Token()
{
Name = "{"
},
new C.Data.Declaration(true)
{
new C.Data.Token()
{
Name = "EPSILON"
}
},
new C.Data.Statement(true)
{
new C.Data.CompoundStatement(true)
{
new C.Data.Statement(true)
{
new C.Data.Token()
{
Name = "EPSILON"
}
},
new C.Data.Statement(true)
{
new C.Data.IntegerAssignment(true)
{
new C.Data.Token()
{
Name = "identifier"
},
new C.Data.Token()
{
Name = "="
},
new C.Data.IntegerExpression(true)
{
new C.Data.AddExpression(true)
{
new C.Data.Token()
{
Name = "("
},
new C.Data.IntegerExpression(true)
{
new C.Data.AddExpression(true)
{
new C.Data.Token()
{
Name = "("
},
new C.Data.IntegerExpression(true)
{
new C.Data.Token()
{
Name = "numeral"
}
},
new C.Data.Token()
{
Name = "+"
},
new C.Data.IntegerExpression(true)
{
new C.Data.Token()
{
Name = "numeral"
}
},
new C.Data.Token()
{
Name = ")"
}
}
},
new C.Data.Token()
{
Name = "+"
},
new C.Data.IntegerExpression(true)
{
new C.Data.Token()
{
Name = "numeral"
}
},
new C.Data.Token()
{
Name = ")"
}
}
}
},
new C.Data.Token()
{
Name = ";"
}
}
}
},
new C.Data.Token()
{
Name = "}"
}
}
}
}
};
// Use this to print the actual tree
//var CCC = c.Accept(new C.Visitors.TreePrintVisitor());
//foreach (var line in CCC)
//{
// Debug.WriteLine(line);
//}
// Call a method to walk the tree's nodes and test if their names are equal
TreeAsserter(c, cExpected);
}
