public void SimpleParseTest()
{
var codes = new[] {
@"<?php
class X {
function static() { }
}",
@"<?php
class enum extends A {
}",
@"<?php
A::E->foo(); // dereferencable class const
",
};
foreach (var code in codes)
{
var sourceUnit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
Assert.IsNotNull(sourceUnit.Ast);
}
}
public void LexerGetNextTokenTest()
{
string path = (string)TestContext.DataRow["files"];
TestErrorSink errorSink = new TestErrorSink();
Lexer lexer = new Lexer(new StreamReader(path), Encoding.UTF8, errorSink,
LanguageFeatures.ShortOpenTags, 0, Lexer.LexicalStates.INITIAL);
string parsed = ParseByPhp(path);
parsed = parsed.Substring(0, parsed.LastIndexOf('-'));
parsed = Regex.Replace(parsed.Replace("\r", " ").Replace("\n", " "), @"\s+", " ");
int i = 0;
string[][] expectedTokens = (
from s in parsed.Split('-')
let num = i++
group s by num / 3 into g
select g.ToArray()
).ToArray();
//List<KeyValuePair<Tokens, string>> l = new List<KeyValuePair<Tokens, string>>();
//Tokens t = Tokens.END;
//while ((t = (Tokens)lexer.GetNextToken()) != Tokens.END)
//{
// l.Add(new KeyValuePair<Tokens, string>(t, lexer.TokenText));
//}
foreach (var expectedToken in expectedTokens)
{
Tokens token = (Tokens)lexer.GetNextToken();
Assert.AreEqual(int.Parse(expectedToken[0]), (int)token, path);
if (token == Tokens.T_VARIABLE || token == Tokens.T_STRING || token == Tokens.T_END_HEREDOC)
{
Assert.AreEqual(expectedToken[2].TrimStart('$'), lexer.TokenValue.Object.ToString());
}
if (token == Tokens.T_DNUMBER)
{
Assert.AreEqual(double.Parse(expectedToken[2], System.Globalization.NumberFormatInfo.InvariantInfo), lexer.TokenValue.Double);
}
if (token == Tokens.T_LNUMBER)
{
Assert.AreEqual(int.Parse(expectedToken[2]), lexer.TokenValue.Integer);
}
//lexer.RestoreCompressedState(lexer.GetCompressedState());
}
Assert.AreEqual(Tokens.EOF, lexer.GetNextToken(), path);
Assert.AreEqual(Tokens.EOF, lexer.GetNextToken(), path);
Assert.AreEqual(Tokens.EOF, lexer.GetNextToken(), path);
Assert.AreEqual(0, errorSink.Errors.Count);
}
public void SimpleParseTest()
{
string code = @"<?php
class X {
function static() { }
}
";
var sourceUnit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
}
public void TokensVisitorTest()
{
string path = (string)TestContext.DataRow["files"];
if (path.Contains("functions1.phpt"))
{
return; // TODO - too slow test
}
string testcontent = File.ReadAllText(path);
var original = testcontent;
var sourceUnit = new TestSourceUnit(original, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
GlobalCode ast = null;
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
ast = sourceUnit.Ast;
if (errors.Count != 0)
{
return; // AST is null or invalid
}
var provider = SourceTokenProviderFactory.CreateProvider(sourceUnit.SourceLexer.AllTokens, original);
var composer = new WhitespaceComposer(provider);
var visitor = new TokenVisitor(new TreeContext(ast), composer, provider);
visitor.VisitElement(ast);
var code = composer.Code;
var result = code.ToString();
//File.WriteAllText(Path.Combine(Directory.GetParent(path).FullName, "original.txt"), original);
//File.WriteAllText(Path.Combine(Directory.GetParent(path).FullName, "result.txt"), result);
//Assert.AreEqual(original.Length, result.Length);
//for (int i = 0; i < original.Length; i++)
//{
// Assert.AreEqual(original[i], result[i]);
//}
Assert.AreEqual(original, result);
var tokens = provider.GetTokens(new Span(0, original.Length)).AsArray();
Assert.AreEqual(tokens.Length, composer.Processed.Count);
for (int i = 0; i < tokens.Length; i++)
{
Assert.AreEqual(tokens[i].Token, composer.Processed[i].Token);
Assert.AreEqual(tokens[i].Span, composer.Processed[i].Span);
}
}
public void MatchTest()
{
var codes = new[] {
@"<?php echo match($x) { 0 => 'hello', 1, 2, 3 => 'world', default => '!', };",
};
foreach (var code in codes)
{
var errors = new TestErrorSink();
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, features: LanguageFeatures.Php80Set);
unit.Parse(new BasicNodesFactory(unit), errors);
Assert.AreEqual(0, errors.Count);
}
}
public void InstanceOfTest()
{
var codes = new[] {
@"<?php echo $x instanceof ((string)$y);",
@"<?php echo $x instanceof $y[0];",
};
foreach (var code in codes)
{
var errors = new TestErrorSink();
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, features: LanguageFeatures.Php80Set);
unit.Parse(new BasicNodesFactory(unit), errors);
Assert.AreEqual(0, errors.Count);
}
}
public void LexerGetNextTokenByLineTest()
{
string path = (string)TestContext.DataRow["files"];
TestErrorSink errorSink = new TestErrorSink();
Lexer lexer = new Lexer(new StreamReader(path), Encoding.UTF8, errorSink,
LanguageFeatures.ShortOpenTags, 0, Lexer.LexicalStates.INITIAL);
Lexer.LexicalStates previousState = Lexer.LexicalStates.INITIAL;
foreach (var line in File.ReadAllLines(path))
{
lexer.Initialize(new StringReader(line + Environment.NewLine), previousState, true, 0);
while (lexer.GetNextToken() != Tokens.EOF)
{
Assert.IsTrue(lexer.TokenSpan.IsValid);
}
previousState = lexer.CurrentLexicalState;
}
}
public void VisitorVisitTests()
{
string path = (string)TestContext.DataRow["files"];
string testcontent = File.ReadAllText(path);
string[] testparts = testcontent.Split(new string[] { "<<<TEST>>>" }, StringSplitOptions.RemoveEmptyEntries);
Assert.IsTrue(testparts.Length >= 2);
var sourceUnit = new CodeSourceUnit(testparts[0], path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new AstCounterFactory(sourceUnit);
var errors = new TestErrorSink();
GlobalCode ast = null;
using (StringReader source_reader = new StringReader(testparts[0]))
{
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
ast = sourceUnit.Ast;
}
if (testparts[1].TrimStart().StartsWith(ParserTests.Errors))
{
var matches = _errorRegex.Matches(testparts[1]);
var knownErrors = matches[0].Groups["Number"].Value.Split(',');
Assert.AreEqual(1, matches.Count, path);
Assert.AreEqual(knownErrors.Length, errors.Count, path);
}
else
{
Assert.AreEqual(0, errors.Count, path);
Assert.IsNotNull(ast);
// check every node has a parent
var checker = new TreeVisitorCheck();
checker.VisitElement(ast);
Assert.AreEqual(factory.CreatedElements.Count, checker.VisitedElements.Count, path);
Assert.AreEqual(factory.ItemCount, checker.ItemCount, path);
Assert.AreEqual(factory.ForeachVarCount, checker.ForeachVarCount, path);
//var dictionary = factory.CreatedElements.GroupBy(t => t.GetType()).ToDictionary(g => g.Key);
}
}
public void VisitorVisitTests()
{
string path = (string)TestContext.DataRow["files"];
string testcontent = File.ReadAllText(path);
string[] testparts = testcontent.Split(new string[] { "<<<TEST>>>" }, StringSplitOptions.RemoveEmptyEntries);
Assert.IsTrue(testparts.Length >= 2);
var sourceUnit = new CodeSourceUnit(testparts[0], path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new AstCounterFactory(sourceUnit);
var errors = new TestErrorSink();
bool expectErrors = testparts[1].TrimStart().StartsWith(ParserTests.Errors);
GlobalCode ast = null;
Parser parser = new Parser();
using (StringReader source_reader = new StringReader(testparts[0]))
{
sourceUnit.Parse(factory, errors);
ast = sourceUnit.Ast;
}
if (expectErrors)
{
Assert.AreEqual(1, errors.Count, path);
}
else
{
Assert.AreEqual(0, errors.Count, path);
// check every node has a parent
var checker = new TreeVisitorCheck();
checker.VisitElement(ast);
Assert.AreEqual(factory.CreatedElements.Count, checker.VisitedElements.Count, path);
Assert.AreEqual(factory.ItemCount, checker.ItemCount, path);
Assert.AreEqual(factory.ForeachVarCount, checker.ForeachVarCount, path);
}
}
public void LexerStringsTest()
{
TestErrorSink errorSink = new TestErrorSink();
Lexer lexer = new Lexer(new StringReader("\"\""), Encoding.UTF8, errorSink,
LanguageFeatures.ShortOpenTags, 0, Lexer.LexicalStates.INITIAL);
var charSet = new[] { new [] { '$', '{', 'n', '\0', '\r', '\n', ' ' },
new [] { '\'', '\\', 'x', 'c', '"', '`', '8', '0' },
new [] { '/', '*', '?', '>', ';' } };
int[] word = new int[5];
char[] text = new char[word.Length];
var states = new Lexer.LexicalStates[] { Lexer.LexicalStates.ST_DOUBLE_QUOTES, Lexer.LexicalStates.ST_SINGLE_QUOTES,
Lexer.LexicalStates.ST_BACKQUOTE, Lexer.LexicalStates.ST_HEREDOC, Lexer.LexicalStates.ST_NOWDOC, Lexer.LexicalStates.ST_COMMENT,
Lexer.LexicalStates.ST_DOC_COMMENT, Lexer.LexicalStates.INITIAL, Lexer.LexicalStates.ST_IN_SCRIPTING };
foreach (var chars in charSet)
{
foreach (var state in states)
{
while (Increment(word, chars.Length))
{
ToArray(word, text, chars);
string line = new string(text);
lexer.Initialize(new StringReader(line), state, true, 0);
Tokens token = Tokens.EOF;
int count = 0;
while ((token = lexer.GetNextToken()) != Tokens.EOF && count++ < 100)
{
Assert.IsTrue(lexer.TokenSpan.IsValid, line);
Assert.IsTrue(lexer.TokenSpan.Length >= 0, line + " - " + state.ToString() + " - " + lexer.TokenSpan.Start.ToString());
}
Assert.IsTrue(count < 100, line);
}
}
}
}
public void ParserParseTest()
{
string path = (string)TestContext.DataRow["files"];
string testcontent = File.ReadAllText(path);
string[] testparts = testcontent.Split(new string[] { "<<<TEST>>>" }, StringSplitOptions.RemoveEmptyEntries);
Assert.IsTrue(testparts.Length >= 2);
var sourceUnit = new CodeSourceUnit(testparts[0], path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
//
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
//
if (testparts[1].TrimStart().StartsWith(Errors))
{
var matches = _errorRegex.Matches(testparts[1]);
var knownErrors = matches[0].Groups["Number"].Value.Split(',');
Assert.AreEqual(1, matches.Count, path);
Assert.AreEqual(knownErrors.Length, errors.Count, path);
int errorid = 0;
for (int i = 0; i < knownErrors.Length; i++)
{
Assert.IsTrue(int.TryParse(knownErrors[i], out errorid), path);
Assert.AreEqual(errorid, errors.Errors[i].Error.Id, path);
Assert.IsNotNull(errors.Errors[i].ToString());
}
testparts[1] = matches[0].Groups["JSON"].Value;
}
else
{
Assert.AreEqual(0, errors.Count, path);
}
Assert.IsNotNull(sourceUnit.Ast);
var serializer = new JsonNodeWriter();
TreeSerializer visitor = new TreeSerializer(serializer);
sourceUnit.Ast.VisitMe(visitor);
Regex rgx = new Regex(@"""Span""[^}]*},?\s*\n?"); // omit Span for more compact testing (position must be verified separately)
string expected = rgx.Replace(testparts[1].Trim().Replace("\r", string.Empty).Replace("\n", string.Empty).Replace(" ", string.Empty), string.Empty);
string actual = rgx.Replace(serializer.ToString().Replace("\r", string.Empty).Replace("\n", string.Empty).Replace(" ", string.Empty), string.Empty);
if (testparts[1].Trim() != "<<<IGNORE>>>")
{
// IMPORTANT - Uncomment to regenerate test data
//File.WriteAllText(path, testparts[0] + "\n<<<TEST>>>\n" + rgx.Replace(serializer.ToString(), string.Empty));
Assert.AreEqual(expected, actual, path);
}
// check every node has a parent
var parentChecker = new ContainingElementCheck();
parentChecker.VisitGlobalCode(sourceUnit.Ast);
// check nodes have correct span corresponding to correct source text
var spanChecker = new NameSpanCheck(testparts[0]);
spanChecker.VisitGlobalCode(sourceUnit.Ast);
}
public void EmptyTokensVisitorTest()
{
string path = (string)TestContext.DataRow["files"];
if (path.Contains("functions1.phpt"))
{
return; // TODO - too slow test
}
string testcontent = File.ReadAllText(path);
var original = testcontent;
if (original.Contains("namespace\\"))
{
return; // TODO - current namespace cannot be decided from AST
}
var sourceUnit = new TestSourceUnit(original, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
GlobalCode ast = sourceUnit.Ast;
if (errors.Count != 0)
{
return; // AST is null or invalid
}
var provider = SourceTokenProviderFactory.CreateEmptyProvider();
var composer = new EmptyComposer(provider);
var visitor = new TokenVisitor(new TreeContext(ast), composer, provider);
visitor.VisitElement(ast);
var code = composer.Code.ToString();
var expectedStr = PrepareString(original);
var actualStr = PrepareString(code);
Assert.AreEqual(expectedStr, actualStr);
var expected = FilterTokens(sourceUnit.SourceLexer.AllTokens);
var actual = FilterTokens(composer.Processed);
Assert.AreEqual(expected.Length, actual.Length);
for (int i = 0; i < Math.Min(expected.Length, actual.Length); i++)
{
if (expected[i].Token == Tokens.T_SEMI && actual[i].Token == Tokens.T_CASE)
{
}
if (expected[i].Token == Tokens.T_LOGICAL_OR && actual[i].Token == Tokens.T_BOOLEAN_OR ||
expected[i].Token == Tokens.T_LOGICAL_AND && actual[i].Token == Tokens.T_BOOLEAN_AND)
{
}
else
{
Assert.AreEqual(expected[i].Token, actual[i].Token);
}
}
sourceUnit = new TestSourceUnit(code, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
var newAst = sourceUnit.Ast;
var serializer = new JsonNodeWriter();
var serializerVisitor = new TreeSerializer(serializer);
ast.VisitMe(visitor);
expectedStr = serializer.ToString();
serializer = new JsonNodeWriter();
serializerVisitor = new TreeSerializer(serializer);
newAst.VisitMe(visitor);
actualStr = serializer.ToString();
Assert.AreEqual(expectedStr, actualStr);
}
