/// <summary>
/// Creates source unit and parses given <paramref name="code"/>.
/// </summary>
/// <param name="code">Source code to be parsed.</param>
/// <param name="filePath">Source file used for error reporting.</param>
/// <param name="factory">Nodes factory and error sink.</param>
/// <param name="errors">Error sink. Can be <c>null</c>.</param>
/// <param name="recovery">Error recovery. Can be <c>null</c>.</param>
/// <param name="features">Optional. Language features.</param>
/// <param name="initialState">
/// Optional. Initial parser state.
/// This allows e.g. to parse PHP code without encapsulating the code into opening and closing tags.</param>
/// <returns>New <see cref="CodeSourceUnit"/> object.</returns>
public static SourceUnit /*!*/ ParseCode(string code, string filePath,
INodesFactory <LangElement, Span> factory = null,
Errors.IErrorSink <Span> errors = null,
Errors.IErrorRecovery recovery = null,
LanguageFeatures features = LanguageFeatures.Basic,
Lexer.LexicalStates initialState = Lexer.LexicalStates.INITIAL)
{
var unit = new CodeSourceUnit(code, filePath, Encoding.UTF8, initialState, features);
if (factory == null)
{
factory = new BasicNodesFactory(unit);
}
if (errors == null)
{
errors = (factory as Errors.IErrorSink <Span>) ?? new EmptyErrorSink <Span>();
}
//var lexer = new Lexer(new StringReader(code), Encoding.UTF8, errors, features, 0, initialState);
unit.Parse(factory, errors, recovery);
unit.Close();
//
return(unit);
}
public void SimpleParseTest()
{
var codes = new[] {
@"<?php
class X {
function static() { }
}",
@"<?php
class enum extends A {
}",
@"<?php
A::E->foo(); // dereferencable class const
",
};
foreach (var code in codes)
{
var sourceUnit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
Assert.IsNotNull(sourceUnit.Ast);
}
}
public PhpChannelRewriter(TreeContext treeContext, ITokenComposer tokenComposer, ISourceTokenProvider sourceTokenProvider, BasicNodesFactory fac, IPolicy policy, List <Channel> inputChannels, List <Channel> outputChannels, List <Channel> sanitizeChannels, SecurityLevel level, bool isOriginalProgram = false)
: base(treeContext, tokenComposer, sourceTokenProvider)
{
_factory = fac;
_policy = policy;
_inputChannels = inputChannels;
_minInputLevel = _inputChannels.Min(ic => ic.Label.Level); //determine lowest ordinal value for all input channels
_outputChannels = outputChannels;
_sanitizeChannels = sanitizeChannels;
_securityLevel = level;
_isOriginalProgram = isOriginalProgram;
}
public void SimpleParseTest()
{
string code = @"<?php
class X {
function static() { }
}
";
var sourceUnit = new CodeSourceUnit(code, "dummy.php", Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
}
public void TokensVisitorTest()
{
string path = (string)TestContext.DataRow["files"];
if (path.Contains("functions1.phpt"))
{
return; // TODO - too slow test
}
string testcontent = File.ReadAllText(path);
var original = testcontent;
var sourceUnit = new TestSourceUnit(original, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
GlobalCode ast = null;
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
ast = sourceUnit.Ast;
if (errors.Count != 0)
{
return; // AST is null or invalid
}
var provider = SourceTokenProviderFactory.CreateProvider(sourceUnit.SourceLexer.AllTokens, original);
var composer = new WhitespaceComposer(provider);
var visitor = new TokenVisitor(new TreeContext(ast), composer, provider);
visitor.VisitElement(ast);
var code = composer.Code;
var result = code.ToString();
//File.WriteAllText(Path.Combine(Directory.GetParent(path).FullName, "original.txt"), original);
//File.WriteAllText(Path.Combine(Directory.GetParent(path).FullName, "result.txt"), result);
//Assert.AreEqual(original.Length, result.Length);
//for (int i = 0; i < original.Length; i++)
//{
// Assert.AreEqual(original[i], result[i]);
//}
Assert.AreEqual(original, result);
var tokens = provider.GetTokens(new Span(0, original.Length)).AsArray();
Assert.AreEqual(tokens.Length, composer.Processed.Count);
for (int i = 0; i < tokens.Length; i++)
{
Assert.AreEqual(tokens[i].Token, composer.Processed[i].Token);
Assert.AreEqual(tokens[i].Span, composer.Processed[i].Span);
}
}
static void Main(string[] args)
{
string code = File.ReadAllText(@"D:\wamp64\www\tachyon\PHPParser\tphp\tphp\a.php");
var sourceUnit = new CodeSourceUnit(code, @"D:\wamp64\www\tachyon\PHPParser\tphp\tphp\a.php", Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var factory = new BasicNodesFactory(sourceUnit);
GlobalCode ast = null;
var errors = new TestErrorSink();
Parser parser = new Parser();
using (StringReader source_reader = new StringReader(code))
{
sourceUnit.Parse(factory, errors);
ast = sourceUnit.Ast;
}
var visitor = new TreeVisitorCheck();
visitor.VisitElement(ast);
Console.Read();
}
public PHPAuditSourceUnit(AuditEnvironment audit_env, string code, FileInfo file) : base(code, file.FullName, Encoding.UTF8)
{
this.ErrorSink = new PHPAuditErrorSink(this.AuditEnvironment);
BasicNodesFactory factory = new BasicNodesFactory(this);
try
{
this.Parse(factory, this.ErrorSink);
if (this.Ast != null)
{
this.DTV = new DeclarationsTreeVisitor();
this.Ast.VisitMe(this.DTV);
}
}
catch (Exception e)
{
this.AuditEnvironment.Error("Parsing file {0} through an exception.", file.FullName);
this.AuditEnvironment.Error(e);
}
}
public void ParserParseTest()
{
string path = (string)TestContext.DataRow["files"];
string testcontent = File.ReadAllText(path);
string[] testparts = testcontent.Split(new string[] { "<<<TEST>>>" }, StringSplitOptions.RemoveEmptyEntries);
Assert.IsTrue(testparts.Length >= 2);
var sourceUnit = new CodeSourceUnit(testparts[0], path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Basic);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
//
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
//
if (testparts[1].TrimStart().StartsWith(Errors))
{
var matches = _errorRegex.Matches(testparts[1]);
var knownErrors = matches[0].Groups["Number"].Value.Split(',');
Assert.AreEqual(1, matches.Count, path);
Assert.AreEqual(knownErrors.Length, errors.Count, path);
int errorid = 0;
for (int i = 0; i < knownErrors.Length; i++)
{
Assert.IsTrue(int.TryParse(knownErrors[i], out errorid), path);
Assert.AreEqual(errorid, errors.Errors[i].Error.Id, path);
Assert.IsNotNull(errors.Errors[i].ToString());
}
testparts[1] = matches[0].Groups["JSON"].Value;
}
else
{
Assert.AreEqual(0, errors.Count, path);
}
Assert.IsNotNull(sourceUnit.Ast);
var serializer = new JsonNodeWriter();
TreeSerializer visitor = new TreeSerializer(serializer);
sourceUnit.Ast.VisitMe(visitor);
Regex rgx = new Regex(@"""Span""[^}]*},?\s*\n?"); // omit Span for more compact testing (position must be verified separately)
string expected = rgx.Replace(testparts[1].Trim().Replace("\r", string.Empty).Replace("\n", string.Empty).Replace(" ", string.Empty), string.Empty);
string actual = rgx.Replace(serializer.ToString().Replace("\r", string.Empty).Replace("\n", string.Empty).Replace(" ", string.Empty), string.Empty);
if (testparts[1].Trim() != "<<<IGNORE>>>")
{
// IMPORTANT - Uncomment to regenerate test data
//File.WriteAllText(path, testparts[0] + "\n<<<TEST>>>\n" + rgx.Replace(serializer.ToString(), string.Empty));
Assert.AreEqual(expected, actual, path);
}
// check every node has a parent
var parentChecker = new ContainingElementCheck();
parentChecker.VisitGlobalCode(sourceUnit.Ast);
// check nodes have correct span corresponding to correct source text
var spanChecker = new NameSpanCheck(testparts[0]);
spanChecker.VisitGlobalCode(sourceUnit.Ast);
}
public void EmptyTokensVisitorTest()
{
string path = (string)TestContext.DataRow["files"];
if (path.Contains("functions1.phpt"))
{
return; // TODO - too slow test
}
string testcontent = File.ReadAllText(path);
var original = testcontent;
if (original.Contains("namespace\\"))
{
return; // TODO - current namespace cannot be decided from AST
}
var sourceUnit = new TestSourceUnit(original, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var factory = new BasicNodesFactory(sourceUnit);
var errors = new TestErrorSink();
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
GlobalCode ast = sourceUnit.Ast;
if (errors.Count != 0)
{
return; // AST is null or invalid
}
var provider = SourceTokenProviderFactory.CreateEmptyProvider();
var composer = new EmptyComposer(provider);
var visitor = new TokenVisitor(new TreeContext(ast), composer, provider);
visitor.VisitElement(ast);
var code = composer.Code.ToString();
var expectedStr = PrepareString(original);
var actualStr = PrepareString(code);
Assert.AreEqual(expectedStr, actualStr);
var expected = FilterTokens(sourceUnit.SourceLexer.AllTokens);
var actual = FilterTokens(composer.Processed);
Assert.AreEqual(expected.Length, actual.Length);
for (int i = 0; i < Math.Min(expected.Length, actual.Length); i++)
{
if (expected[i].Token == Tokens.T_SEMI && actual[i].Token == Tokens.T_CASE)
{
}
if (expected[i].Token == Tokens.T_LOGICAL_OR && actual[i].Token == Tokens.T_BOOLEAN_OR ||
expected[i].Token == Tokens.T_LOGICAL_AND && actual[i].Token == Tokens.T_BOOLEAN_AND)
{
}
else
{
Assert.AreEqual(expected[i].Token, actual[i].Token);
}
}
sourceUnit = new TestSourceUnit(code, path, Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
sourceUnit.Parse(factory, errors, new TestErrorRecovery());
var newAst = sourceUnit.Ast;
var serializer = new JsonNodeWriter();
var serializerVisitor = new TreeSerializer(serializer);
ast.VisitMe(visitor);
expectedStr = serializer.ToString();
serializer = new JsonNodeWriter();
serializerVisitor = new TreeSerializer(serializer);
newAst.VisitMe(visitor);
actualStr = serializer.ToString();
Assert.AreEqual(expectedStr, actualStr);
}
public TransformationResult Transform(string content, IPolicy policy)
{
var result = new TransformationResult();
var sourceUnit = new CodeSourceUnit(content, filename, System.Text.Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var nodesFactory = new BasicNodesFactory(sourceUnit);
var errors = new PhpErrorSink();
sourceUnit.Parse(nodesFactory, errors);
GlobalCode ast = sourceUnit.Ast;
if (errors.Count != 0)
{
ReportErrors(errors, result, content);
return(result); // AST is null or invalid
}
//collect channel information from source code
var provider = SourceTokenProviderFactory.CreateEmptyProvider();
var collectorComposer = new PhpTokenComposer(provider);
var collector = new PhpChannelCollector(policy, new TreeContext(ast), collectorComposer, provider);
collector.VisitElement(ast);
result.InputChannels.AddRange(collector.InputChannels);
result.OutputChannels.AddRange(collector.OutputChannels);
result.SanitizeChannels.AddRange(collector.SanitizeChannels);
//if there are no output or input channels found in the code, it makes no sense to transform it
if (result.OutputChannels.Count == 0 || result.InputChannels.Count == 0)
{
return(result);
}
var levels = collector.GetDistinctSecurityLevels().OrderByDescending(sl => sl.Level);
//append a sanitize transformation if there are sanitize channels
var lowestInputLevel = result.InputChannels.Min(sc => sc.Label.Level);
if (result.SanitizeChannels.Any())
{
var sanSourceUnit = new CodeSourceUnit(content, filename, System.Text.Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var sanNodesFactory = new BasicNodesFactory(sourceUnit);
var sanErrors = new PhpErrorSink();
sanSourceUnit.Parse(sanNodesFactory, sanErrors);
GlobalCode sanAst = sanSourceUnit.Ast;
if (sanErrors.Count != 0)
{
return(result); // AST is null or invalid
}
var pSanitize = new CodeTransformation();
pSanitize.Kind = TransformationKind.Sanitize;
pSanitize.SecurityLevel = new SecurityLevel()
{
Level = lowestInputLevel - 1, Name = "PS"
};
var composer = new PhpTokenComposer(provider);
var rewriter = new PhpChannelRewriter(new TreeContext(sanAst), composer, provider, nodesFactory, policy, collector.InputChannels, collector.OutputChannels, collector.SanitizeChannels, pSanitize.SecurityLevel);
rewriter.VisitElement(sanAst);
pSanitize.Code = composer.Code.ToString();
result.CodeTransformations.Add(pSanitize);
}
//create code version for each security level
foreach (var level in levels)
{
var levelSourceUnit = new CodeSourceUnit(content, filename, System.Text.Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var levelNodesFactory = new BasicNodesFactory(levelSourceUnit);
var levelerrors = new PhpErrorSink();
levelSourceUnit.Parse(nodesFactory, levelerrors);
GlobalCode levelAst = levelSourceUnit.Ast;
if (levelerrors.Count != 0)
{
return(result); // AST is null or invalid
}
var version = new CodeTransformation();
version.Kind = TransformationKind.Default;
version.SecurityLevel = level;
var composer = new PhpTokenComposer(provider);
var rewriter = new PhpChannelRewriter(new TreeContext(levelAst), composer, provider, levelNodesFactory, policy, collector.InputChannels, collector.OutputChannels, collector.SanitizeChannels, level);
rewriter.VisitElement(levelAst);
version.Code = composer.Code.ToString();
result.CodeTransformations.Add(version);
}
//create P version
var poSourceUnit = new CodeSourceUnit(content, filename, System.Text.Encoding.UTF8, Lexer.LexicalStates.INITIAL, LanguageFeatures.Php71Set);
var poNodesFactory = new BasicNodesFactory(poSourceUnit);
var poErrors = new PhpErrorSink();
poSourceUnit.Parse(poNodesFactory, poErrors);
GlobalCode poAst = poSourceUnit.Ast;
var po = new CodeTransformation();
po.Kind = TransformationKind.Original;
var poComposer = new PhpTokenComposer(provider);
po.SecurityLevel = new SecurityLevel()
{
Level = lowestInputLevel, Name = "P'"
};
var poRewriter = new PhpChannelRewriter(new TreeContext(poAst), poComposer, provider, poNodesFactory, policy, collector.InputChannels, collector.OutputChannels, collector.SanitizeChannels, po.SecurityLevel, isOriginalProgram: true);
poRewriter.VisitElement(poAst);
po.Code = poComposer.Code.ToString();
result.CodeTransformations.Add(po);
return(result);
}
