public static PhpSyntaxTree ParseCode(
string content,
PhpParseOptions parseOptions,
PhpParseOptions scriptParseOptions,
string fname)
{
// TODO: new parser implementation based on Roslyn
// TODO: file.IsScript ? scriptParseOptions : parseOptions
var unit = new CodeSourceUnit(
content, fname, Encoding.UTF8,
(parseOptions.Kind == SourceCodeKind.Regular) ? Lexer.LexicalStates.INITIAL : Lexer.LexicalStates.ST_IN_SCRIPTING);
var result = new PhpSyntaxTree(unit);
var errorSink = new ErrorSink(result);
var factory = new NodesFactory(unit);
//
unit.Parse(factory, errorSink);
//
result.Diagnostics = errorSink.Diagnostics;
result.Lambdas = factory.Lambdas.AsImmutableSafe();
result.Types = factory.Types.AsImmutableSafe();
result.Functions = factory.Functions.AsImmutableSafe();
result.YieldNodes = factory.YieldNodes.AsImmutableSafe();
result.Root = factory.Root;
//
return(result);
}
public void SimpleParseTest()
{
var codes = new[] {
@"<?php
class X {
function static() { }
}",
@"<?php
class enum extends A {
}",
@"<?php
A::E->foo(); // dereferencable class const
",
};
foreach (var code in codes)
{
var sourceUnit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
Assert.IsNotNull(sourceUnit.Ast);
}
}
public void PhpDocAttributesTest()
{
var codes = new[] {
@"<?php
/** phpdoc */
#[ClassName(1,2,3)]
class X {
/** phpdoc */
#[ClassName]
function foo() { }
}",
};
foreach (var code in codes)
{
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8);
unit.Parse(new BasicNodesFactory(unit), null);
foreach (var tdecl in unit.Ast.TraverseNamedTypeDeclarations())
{
Assert.IsNotNull(tdecl.PHPDoc);
foreach (var m in tdecl.Members.OfType <MethodDecl>())
{
Assert.IsNotNull(m.PHPDoc);
}
}
}
}
public void LexerConstructorTest()
{
string path = (string)TestContext.DataRow["files"];
SourceUnit sourceUnit = new CodeSourceUnit(File.ReadAllText(path), path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
ITokenProvider <SemanticValueType, Span> lexer = new Lexer(new StreamReader(path), Encoding.UTF8, new TestErrorSink(),
LanguageFeatures.ShortOpenTags, 0, Lexer.LexicalStates.INITIAL);
Assert.AreNotEqual(null, lexer);
}
public void StringLiteralTest()
{
var codes = new[] {
@"<?php echo ""\\test"";",
};
foreach (var code in codes)
{
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8);
unit.Parse(new BasicNodesFactory(unit), null);
Assert.IsNotNull(unit.Ast);
}
}
public void AttributesTest()
{
var codes = new[] {
@"<?php #[ClassName(1,2,3)]class X { }",
@"<?php #[ClassName]class X { }",
};
foreach (var code in codes)
{
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8);
unit.Parse(new BasicNodesFactory(unit), null);
}
}
public void NamedArgTest()
{
var codes = new[] {
@"<?php array_fill(start_index: 0, num: 100, value: 50);",
@"<?php implode(separator: '.', array: []);"
};
foreach (var code in codes)
{
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8);
unit.Parse(new BasicNodesFactory(unit), null);
}
}
public void ReturnTypeTest()
{
var codes = new[] {
@"<?php class X { function foo() : static { } }"
};
foreach (var code in codes)
{
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8);
unit.Parse(new BasicNodesFactory(unit), null);
Assert.IsNotNull(unit.Ast);
}
}
public void ArrowFuncTest()
{
var codes = new[] {
@"<?php $foo = fn() => 1;",
@"<?php $foo = static fn() => 1;"
};
foreach (var code in codes)
{
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8);
unit.Parse(new BasicNodesFactory(unit), null);
}
}
public void AliasesTest()
{
var codes = new[] {
@"<?php use A\{X,Y,};", // <-- trailing comma
};
foreach (var code in codes)
{
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8);
unit.Parse(new BasicNodesFactory(unit), null);
Assert.IsNotNull(unit.Ast);
}
}
public static SyntaxTree ParseCode(Parser parser, Context context, string content_string, string filename)
{
if (filename == null)
{
throw new ArgumentNullException(nameof(filename));
}
content_string = content_string.Replace("\r", "").Replace("else if", "elseif").Trim();
// TODO: new parser implementation based on Roslyn
// TODO: file.IsScript ? scriptParseOptions : parseOptions
var unit = new CodeSourceUnit(
content_string,
filename,
Encoding.UTF8,
Lexer.LexicalStates.INITIAL,
LanguageFeatures.Php73Set | LanguageFeatures.ShortOpenTags
);
var result = new SyntaxTree(unit);
var errorSink = new PhpErrorSink(result);
var factory = new NodesFactory(unit, context.Defines);
//
unit.Parse(factory, errorSink);
//
result.Diagnostics = errorSink.Diagnostics.Select(d => (IDiagnostic)d).ToImmutableArray();
result.Lambdas = factory.Lambdas.AsImmutableSafe();
result.Types = factory.Types.AsImmutableSafe();
result.Functions = factory.Functions.AsImmutableSafe();
result.YieldNodes = factory.YieldNodes.AsImmutableSafe();
if (factory.Root != null)
{
result.Root = factory.Root;
}
else
{
// Parser leaves factory.Root to null in the case of syntax errors -> create a proxy syntax node
var fullSpan = new Devsense.PHP.Text.Span(0, unit.Code.Length);
result.Root = new GlobalCode(fullSpan, ImmutableArray <Statement> .Empty, unit);
}
result.RootExpression = parser.Parse(result.Root);
return(result);
}
public void SimpleParseTest()
{
string code = @"<?php
class X {
function static() { }
}
";
var sourceUnit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
}
private static SourceUnit ParseFile(
TextWriter consoleOutput,
PhpParseOptions parseOptions,
PhpParseOptions scriptParseOptions,
SourceText content,
CommandLineSourceFile file)
{
// TODO: new parser implementation based on Roslyn
// TODO: file.IsScript ? scriptParseOptions : parseOptions
var tree = CodeSourceUnit.ParseCode(content.ToString(), file.Path, new TextErrorSink(consoleOutput));
return(tree);
}
public void MatchTest()
{
var codes = new[] {
@"<?php echo match($x) { 0 => 'hello', 1, 2, 3 => 'world', default => '!', };",
};
foreach (var code in codes)
{
var errors = new TestErrorSink();
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, features: LanguageFeatures.Php80Set);
unit.Parse(new BasicNodesFactory(unit), errors);
Assert.AreEqual(0, errors.Count);
}
}
public static PhpSyntaxTree ParseCode(
string content,
PhpParseOptions parseOptions,
PhpParseOptions scriptParseOptions,
string fname)
{
if (fname == null)
{
throw new ArgumentNullException(nameof(fname));
}
// TODO: new parser implementation based on Roslyn
// TODO: file.IsScript ? scriptParseOptions : parseOptions
var unit = new CodeSourceUnit(
content, fname, Encoding.UTF8,
(parseOptions.Kind == SourceCodeKind.Regular) ? Lexer.LexicalStates.INITIAL : Lexer.LexicalStates.ST_IN_SCRIPTING, GetLanguageFeatures(parseOptions));
var result = new PhpSyntaxTree(unit);
var errorSink = new ErrorSink(result);
var factory = new NodesFactory(unit, parseOptions.Defines);
//
unit.Parse(factory, errorSink);
//
result.Diagnostics = errorSink.Diagnostics;
result.Lambdas = factory.Lambdas.AsImmutableSafe();
result.Types = factory.Types.AsImmutableSafe();
result.Functions = factory.Functions.AsImmutableSafe();
result.YieldNodes = factory.YieldNodes.AsImmutableSafe();
if (factory.Root != null)
{
result.Root = factory.Root;
}
else
{
// Parser leaves factory.Root to null in the case of syntax errors -> create a proxy syntax node
var fullSpan = new Devsense.PHP.Text.Span(0, unit.Code.Length);
result.Root = new GlobalCode(fullSpan, ImmutableArray <Statement> .Empty, unit);
}
//
return(result);
}
public void InstanceOfTest()
{
var codes = new[] {
@"<?php echo $x instanceof ((string)$y);",
@"<?php echo $x instanceof $y[0];",
};
foreach (var code in codes)
{
var errors = new TestErrorSink();
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, features: LanguageFeatures.Php80Set);
unit.Parse(new BasicNodesFactory(unit), errors);
Assert.AreEqual(0, errors.Count);
}
}
public void EnumerationTest()
{
var codes = new[] {
@"<?php enum Suit { case Hearts; case Diamonds; case Clubs; case Spades; }",
@"<?php enum Suit: int { case Hearts = 1; case Diamonds = 2; case Clubs = 3; case Spades = 4; }",
};
foreach (var code in codes)
{
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, features: LanguageFeatures.Php81Set);
unit.Parse(new BasicNodesFactory(unit), null);
Assert.IsNotNull(unit.Ast);
Assert.IsInstanceOfType(unit.Ast.Statements[0], typeof(NamedTypeDecl));
}
}
static void Main(string[] args)
{
string code = File.ReadAllText(@"D:\wamp64\www\tachyon\PHPParser\tphp\tphp\a.php");
var sourceUnit = new CodeSourceUnit(code, @"D:\wamp64\www\tachyon\PHPParser\tphp\tphp\a.php", Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var factory = new BasicNodesFactory(sourceUnit);
GlobalCode ast = null;
var errors = new TestErrorSink();
Parser parser = new Parser();
using (StringReader source_reader = new StringReader(code))
{
sourceUnit.Parse(factory, errors);
ast = sourceUnit.Ast;
}
var visitor = new TreeVisitorCheck();
visitor.VisitElement(ast);
Console.Read();
}
public static PhpSyntaxTree ParseCode(
string content,
PhpParseOptions parseOptions,
PhpParseOptions scriptParseOptions,
string fname)
{
// TODO: new parser implementation based on Roslyn
// TODO: file.IsScript ? scriptParseOptions : parseOptions
var unit = new CodeSourceUnit(content.ToString(), fname, Encoding.UTF8);
var result = new PhpSyntaxTree(unit);
var errorSink = new ErrorSink(result);
unit.Parse(new BasicNodesFactory(unit), errorSink);
result.Diagnostics = errorSink.Diagnostics;
return(result);
}
public void HeredocTest()
{
var codes = new[] {
@"<?php
$x = <<<XXX
/**
* text
*/
XXX;",
};
foreach (var code in codes)
{
var unit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8);
unit.Parse(new BasicNodesFactory(unit), null);
Assert.IsNotNull(unit.Ast);
}
}
public void VisitorVisitTests()
{
string path = (string)TestContext.DataRow["files"];
string testcontent = File.ReadAllText(path);
string[] testparts = testcontent.Split(new string[] { "<<<TEST>>>" }, StringSplitOptions.RemoveEmptyEntries);
Assert.IsTrue(testparts.Length >= 2);
var sourceUnit = new CodeSourceUnit(testparts[0], path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new AstCounterFactory(sourceUnit);
var errors = new TestErrorSink();
GlobalCode ast = null;
using (StringReader source_reader = new StringReader(testparts[0]))
{
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
ast = sourceUnit.Ast;
}
if (testparts[1].TrimStart().StartsWith(ParserTests.Errors))
{
var matches = _errorRegex.Matches(testparts[1]);
var knownErrors = matches[0].Groups["Number"].Value.Split(',');
Assert.AreEqual(1, matches.Count, path);
Assert.AreEqual(knownErrors.Length, errors.Count, path);
}
else
{
Assert.AreEqual(0, errors.Count, path);
Assert.IsNotNull(ast);
// check every node has a parent
var checker = new TreeVisitorCheck();
checker.VisitElement(ast);
Assert.AreEqual(factory.CreatedElements.Count, checker.VisitedElements.Count, path);
Assert.AreEqual(factory.ItemCount, checker.ItemCount, path);
Assert.AreEqual(factory.ForeachVarCount, checker.ForeachVarCount, path);
//var dictionary = factory.CreatedElements.GroupBy(t => t.GetType()).ToDictionary(g => g.Key);
}
}
public void VisitorVisitTests()
{
string path = (string)TestContext.DataRow["files"];
string testcontent = File.ReadAllText(path);
string[] testparts = testcontent.Split(new string[] { "<<<TEST>>>" }, StringSplitOptions.RemoveEmptyEntries);
Assert.IsTrue(testparts.Length >= 2);
var sourceUnit = new CodeSourceUnit(testparts[0], path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new AstCounterFactory(sourceUnit);
var errors = new TestErrorSink();
bool expectErrors = testparts[1].TrimStart().StartsWith(ParserTests.Errors);
GlobalCode ast = null;
Parser parser = new Parser();
using (StringReader source_reader = new StringReader(testparts[0]))
{
sourceUnit.Parse(factory, errors);
ast = sourceUnit.Ast;
}
if (expectErrors)
{
Assert.AreEqual(1, errors.Count, path);
}
else
{
Assert.AreEqual(0, errors.Count, path);
// check every node has a parent
var checker = new TreeVisitorCheck();
checker.VisitElement(ast);
Assert.AreEqual(factory.CreatedElements.Count, checker.VisitedElements.Count, path);
Assert.AreEqual(factory.ItemCount, checker.ItemCount, path);
Assert.AreEqual(factory.ForeachVarCount, checker.ForeachVarCount, path);
}
}
private static void ProcessTestsXml(Pchp.Core.Context ctx, string source, string path, ITestCaseDiscoverySink discoverySink)
{
var testsEl = XElement.Load(path);
foreach (var testCaseClassEl in testsEl.Descendants("testCaseClass"))
{
var phpClassName = testCaseClassEl.Attribute("name").Value;
var classInfo = ctx.GetDeclaredType(phpClassName, autoload: false);
var filePath = Path.GetFullPath(Path.Combine(ctx.RootPath, classInfo.RelativePath));
// The file might not exist as PHPUnit may mistakenly select classes and methods in its own PHAR,
// namely PHPUnit\Framework\WarningTestCase::Warning (happened with PHPUnit 7.5.9)
var unit =
File.Exists(filePath)
? CodeSourceUnit.ParseCode(File.ReadAllText(filePath), filePath)
: null;
foreach (var testCaseMethodEl in testCaseClassEl.Descendants("testCaseMethod"))
{
var methodName = testCaseMethodEl.Attribute("name").Value;
var testName = PhpUnitHelper.GetTestNameFromPhp(classInfo, methodName);
var testCase = new TestCase(testName, PhpUnitTestExecutor.ExecutorUri, source);
if (unit != null)
{
testCase.DisplayName = $"{classInfo.Name}::{methodName}";
testCase.CodeFilePath = filePath;
testCase.LineNumber = GetLineNumber(unit, phpClassName, methodName);
}
;
ProcessTraits(testCase, testCaseMethodEl.Attribute("groups")?.Value);
discoverySink.SendTestCase(testCase);
}
}
}
private PhpSyntaxTree(CodeSourceUnit source)
{
Contract.ThrowIfNull(source);
_source = source;
}
private SyntaxTree(CodeSourceUnit source)
{
_source = source ?? throw new ArgumentNullException(nameof(source));
}
public void ParserParseTest()
{
string path = (string)TestContext.DataRow["files"];
string testcontent = File.ReadAllText(path);
string[] testparts = testcontent.Split(new string[] { "<<<TEST>>>" }, StringSplitOptions.RemoveEmptyEntries);
Assert.IsTrue(testparts.Length >= 2);
var sourceUnit = new CodeSourceUnit(testparts[0], path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
//
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
//
if (testparts[1].TrimStart().StartsWith(Errors))
{
var matches = _errorRegex.Matches(testparts[1]);
var knownErrors = matches[0].Groups["Number"].Value.Split(',');
Assert.AreEqual(1, matches.Count, path);
Assert.AreEqual(knownErrors.Length, errors.Count, path);
int errorid = 0;
for (int i = 0; i < knownErrors.Length; i++)
{
Assert.IsTrue(int.TryParse(knownErrors[i], out errorid), path);
Assert.AreEqual(errorid, errors.Errors[i].Error.Id, path);
Assert.IsNotNull(errors.Errors[i].ToString());
}
testparts[1] = matches[0].Groups["JSON"].Value;
}
else
{
Assert.AreEqual(0, errors.Count, path);
}
Assert.IsNotNull(sourceUnit.Ast);
var serializer = new JsonNodeWriter();
TreeSerializer visitor = new TreeSerializer(serializer);
sourceUnit.Ast.VisitMe(visitor);
Regex rgx = new Regex(@"""Span""[^}]*},?\s*\n?"); // omit Span for more compact testing (position must be verified separately)
string expected = rgx.Replace(testparts[1].Trim().Replace("\r", string.Empty).Replace("\n", string.Empty).Replace(" ", string.Empty), string.Empty);
string actual = rgx.Replace(serializer.ToString().Replace("\r", string.Empty).Replace("\n", string.Empty).Replace(" ", string.Empty), string.Empty);
if (testparts[1].Trim() != "<<<IGNORE>>>")
{
// IMPORTANT - Uncomment to regenerate test data
//File.WriteAllText(path, testparts[0] + "\n<<<TEST>>>\n" + rgx.Replace(serializer.ToString(), string.Empty));
Assert.AreEqual(expected, actual, path);
}
// check every node has a parent
var parentChecker = new ContainingElementCheck();
parentChecker.VisitGlobalCode(sourceUnit.Ast);
// check nodes have correct span corresponding to correct source text
var spanChecker = new NameSpanCheck(testparts[0]);
spanChecker.VisitGlobalCode(sourceUnit.Ast);
}
public TransformationResult Transform(string content, IPolicy policy)
{
var result = new TransformationResult();
var sourceUnit = new CodeSourceUnit(content, filename, System.Text.Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var nodesFactory = new BasicNodesFactory(sourceUnit);
var errors = new PhpErrorSink();
sourceUnit.Parse(nodesFactory, errors);
GlobalCode ast = sourceUnit.Ast;
if (errors.Count != 0)
{
ReportErrors(errors, result, content);
return(result); // AST is null or invalid
}
//collect channel information from source code
var provider = SourceTokenProviderFactory.CreateEmptyProvider();
var collectorComposer = new PhpTokenComposer(provider);
var collector = new PhpChannelCollector(policy, new TreeContext(ast), collectorComposer, provider);
collector.VisitElement(ast);
result.InputChannels.AddRange(collector.InputChannels);
result.OutputChannels.AddRange(collector.OutputChannels);
result.SanitizeChannels.AddRange(collector.SanitizeChannels);
//if there are no output or input channels found in the code, it makes no sense to transform it
if (result.OutputChannels.Count == 0 || result.InputChannels.Count == 0)
{
return(result);
}
var levels = collector.GetDistinctSecurityLevels().OrderByDescending(sl => sl.Level);
//append a sanitize transformation if there are sanitize channels
var lowestInputLevel = result.InputChannels.Min(sc => sc.Label.Level);
if (result.SanitizeChannels.Any())
{
var sanSourceUnit = new CodeSourceUnit(content, filename, System.Text.Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var sanNodesFactory = new BasicNodesFactory(sourceUnit);
var sanErrors = new PhpErrorSink();
sanSourceUnit.Parse(sanNodesFactory, sanErrors);
GlobalCode sanAst = sanSourceUnit.Ast;
if (sanErrors.Count != 0)
{
return(result); // AST is null or invalid
}
var pSanitize = new CodeTransformation();
pSanitize.Kind = TransformationKind.Sanitize;
pSanitize.SecurityLevel = new SecurityLevel()
{
Level = lowestInputLevel - 1, Name = "PS"
};
var composer = new PhpTokenComposer(provider);
var rewriter = new PhpChannelRewriter(new TreeContext(sanAst), composer, provider, nodesFactory, policy, collector.InputChannels, collector.OutputChannels, collector.SanitizeChannels, pSanitize.SecurityLevel);
rewriter.VisitElement(sanAst);
pSanitize.Code = composer.Code.ToString();
result.CodeTransformations.Add(pSanitize);
}
//create code version for each security level
foreach (var level in levels)
{
var levelSourceUnit = new CodeSourceUnit(content, filename, System.Text.Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var levelNodesFactory = new BasicNodesFactory(levelSourceUnit);
var levelerrors = new PhpErrorSink();
levelSourceUnit.Parse(nodesFactory, levelerrors);
GlobalCode levelAst = levelSourceUnit.Ast;
if (levelerrors.Count != 0)
{
return(result); // AST is null or invalid
}
var version = new CodeTransformation();
version.Kind = TransformationKind.Default;
version.SecurityLevel = level;
var composer = new PhpTokenComposer(provider);
var rewriter = new PhpChannelRewriter(new TreeContext(levelAst), composer, provider, levelNodesFactory, policy, collector.InputChannels, collector.OutputChannels, collector.SanitizeChannels, level);
rewriter.VisitElement(levelAst);
version.Code = composer.Code.ToString();
result.CodeTransformations.Add(version);
}
//create P version
var poSourceUnit = new CodeSourceUnit(content, filename, System.Text.Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var poNodesFactory = new BasicNodesFactory(poSourceUnit);
var poErrors = new PhpErrorSink();
poSourceUnit.Parse(poNodesFactory, poErrors);
GlobalCode poAst = poSourceUnit.Ast;
var po = new CodeTransformation();
po.Kind = TransformationKind.Original;
var poComposer = new PhpTokenComposer(provider);
po.SecurityLevel = new SecurityLevel()
{
Level = lowestInputLevel, Name = "P'"
};
var poRewriter = new PhpChannelRewriter(new TreeContext(poAst), poComposer, provider, poNodesFactory, policy, collector.InputChannels, collector.OutputChannels, collector.SanitizeChannels, po.SecurityLevel, isOriginalProgram: true);
poRewriter.VisitElement(poAst);
po.Code = poComposer.Code.ToString();
result.CodeTransformations.Add(po);
return(result);
}
