public void TokensVisitorTest() { string path = (string)TestContext.DataRow["files"]; if (path.Contains("functions1.phpt")) { return; // TODO - too slow test } string testcontent = File.ReadAllText(path); var original = testcontent; var sourceUnit = new TestSourceUnit(original, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set); var factory = new BasicNodesFactory(sourceUnit); var errors = new TestErrorSink(); GlobalCode ast = null; sourceUnit.Parse(factory, errors, new TestErrorRecovery()); ast = sourceUnit.Ast; if (errors.Count != 0) { return; // AST is null or invalid } var provider = SourceTokenProviderFactory.CreateProvider(sourceUnit.SourceLexer.AllTokens, original); var composer = new WhitespaceComposer(provider); var visitor = new TokenVisitor(new TreeContext(ast), composer, provider); visitor.VisitElement(ast); var code = composer.Code; var result = code.ToString(); //File.WriteAllText(Path.Combine(Directory.GetParent(path).FullName, "original.txt"), original); //File.WriteAllText(Path.Combine(Directory.GetParent(path).FullName, "result.txt"), result); //Assert.AreEqual(original.Length, result.Length); //for (int i = 0; i < original.Length; i++) //{ // Assert.AreEqual(original[i], result[i]); //} Assert.AreEqual(original, result); var tokens = provider.GetTokens(new Span(0, original.Length)).AsArray(); Assert.AreEqual(tokens.Length, composer.Processed.Count); for (int i = 0; i < tokens.Length; i++) { Assert.AreEqual(tokens[i].Token, composer.Processed[i].Token); Assert.AreEqual(tokens[i].Span, composer.Processed[i].Span); } }
public void EmptyTokensVisitorTest() { string path = (string)TestContext.DataRow["files"]; if (path.Contains("functions1.phpt")) { return; // TODO - too slow test } string testcontent = File.ReadAllText(path); var original = testcontent; if (original.Contains("namespace\\")) { return; // TODO - current namespace cannot be decided from AST } var sourceUnit = new TestSourceUnit(original, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set); var factory = new BasicNodesFactory(sourceUnit); var errors = new TestErrorSink(); sourceUnit.Parse(factory, errors, new TestErrorRecovery()); GlobalCode ast = sourceUnit.Ast; if (errors.Count != 0) { return; // AST is null or invalid } var provider = SourceTokenProviderFactory.CreateEmptyProvider(); var composer = new EmptyComposer(provider); var visitor = new TokenVisitor(new TreeContext(ast), composer, provider); visitor.VisitElement(ast); var code = composer.Code.ToString(); var expectedStr = PrepareString(original); var actualStr = PrepareString(code); Assert.AreEqual(expectedStr, actualStr); var expected = FilterTokens(sourceUnit.SourceLexer.AllTokens); var actual = FilterTokens(composer.Processed); Assert.AreEqual(expected.Length, actual.Length); for (int i = 0; i < Math.Min(expected.Length, actual.Length); i++) { if (expected[i].Token == Tokens.T_SEMI && actual[i].Token == Tokens.T_CASE) { } if (expected[i].Token == Tokens.T_LOGICAL_OR && actual[i].Token == Tokens.T_BOOLEAN_OR || expected[i].Token == Tokens.T_LOGICAL_AND && actual[i].Token == Tokens.T_BOOLEAN_AND) { } else { Assert.AreEqual(expected[i].Token, actual[i].Token); } } sourceUnit = new TestSourceUnit(code, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set); sourceUnit.Parse(factory, errors, new TestErrorRecovery()); var newAst = sourceUnit.Ast; var serializer = new JsonNodeWriter(); var serializerVisitor = new TreeSerializer(serializer); ast.VisitMe(visitor); expectedStr = serializer.ToString(); serializer = new JsonNodeWriter(); serializerVisitor = new TreeSerializer(serializer); newAst.VisitMe(visitor); actualStr = serializer.ToString(); Assert.AreEqual(expectedStr, actualStr); }