public void Render(RootUst ust)
{
try
{
var renderer = new StageDotRenderer
{
RenderStages = Stages,
IncludeHiddenTokens = IncludeHiddenTokens,
RenderDirection = RenderDirection
};
string dotGraph = renderer.Render(ust);
string fileName =
(!string.IsNullOrEmpty(Path.GetFileName(ust.SourceFile.Name)) ? ust.SourceFile.Name + "." : "")
+ "ust";
var graph = new GraphvizGraph(dotGraph)
{
OutputFormat = RenderFormat,
Logger = Logger
};
graph.Render(Path.Combine(DumpDir, fileName));
}
catch (Exception ex) when(!(ex is ThreadAbortException))
{
Logger.LogError(ex);
}
}
public RootUst Convert(ParseTree langParseTree)
{
var aspxParseTree = (AspxParseTree)langParseTree;
try
{
RootUst result;
sourceFile = langParseTree.SourceFile;
Ust visited = aspxParseTree.Root.Accept(this);
if (visited is RootUst rootUst)
{
result = rootUst;
}
else
{
result = new RootUst(sourceFile, Language);
result.Node = visited;
}
result.FillAscendants();
return(result);
}
catch (Exception ex) when(!(ex is ThreadAbortException))
{
Logger.LogError(new ConversionException(aspxParseTree.SourceFile, ex));
return(null);
}
}
public static byte[] Serialize(RootUst rootUst, bool isLineColumn, bool compress, ILogger logger,
out int writeSize)
{
var textSpanSerializer = TextSpanSerializer.CreateWriter();
textSpanSerializer.IsLineColumn = isLineColumn;
textSpanSerializer.Logger = logger;
var sourceFileSerializer = FileSerializer.CreateWriter();
sourceFileSerializer.Logger = logger;
var rootUstSerializer = new RootUstMessagePackSerializer
{
textSpanSerializer = textSpanSerializer,
fileSerializer = sourceFileSerializer
};
if (buffer == null)
{
buffer = new byte[ushort.MaxValue + 1];
}
writeSize = rootUstSerializer.SerializeRootUst(ref buffer, 0, rootUst);
if (compress)
{
var result = LZ4MessagePackSerializer.ToLZ4Binary(new ArraySegment <byte>(buffer, 0, writeSize));
return(result);
}
return(buffer);
}
public void Convert_JavaArrayInitialization()
{
var sourceRep = new MemorySourceRepository(
"class ArrayInitialization {\r\n" +
"public void init() {\r\n" +
"int[] arr1 = new int[] { 1, 2, 3 };\r\n" +
"int[][] arr2 = new int[1][2];\r\n" +
"int[][] arr3 = new int[1][];\r\n" +
"}\r\n" +
"}",
"ArrayInitialization.java"
);
RootUst ust = null;
var workflow = new Workflow(sourceRep, stage: Stage.Ust);
workflow.UstConverted += (sender, rootUst) => ust = rootUst;
workflow.Process();
var intType = new TypeToken("int");
var arrayData = new List <Tuple <List <Expression>, List <Expression> > >();
// new int[] { 1, 2, 3 };
arrayData.Add(new Tuple <List <Expression>, List <Expression> >(
Enumerable.Range(1, 3).Select(num => new IntLiteral(num)).ToList <Expression>(),
new List <Expression> {
new IntLiteral(0)
}
));
// new int[1][2];
arrayData.Add(new Tuple <List <Expression>, List <Expression> >(
null,
new List <Expression> {
new IntLiteral(1), new IntLiteral(2)
}
));
// new int[1][];
arrayData.Add(new Tuple <List <Expression>, List <Expression> >(
null,
new List <Expression> {
new IntLiteral(1), new IntLiteral(0)
}
));
for (var i = 0; i < arrayData.Count; i++)
{
var data = arrayData[i];
var arrayCreationExpression = new ArrayCreationExpression
{
Type = intType,
Initializers = data.Item1,
Sizes = data.Item2
};
bool exist = ust.AnyDescendantOrSelf(node => node.Equals(arrayCreationExpression));
Assert.IsTrue(exist, "Test failed on " + i + " iteration.");
}
}
public static byte[] Serialize(RootUst rootUst, bool isLineColumn, bool compress, ILogger logger)
{
byte[] result = Serialize(rootUst, isLineColumn, compress, logger, out int writeSize);
if (compress)
{
return(result);
}
return(MessagePackBinary.FastCloneWithResize(result, writeSize));
}
private Ust ParseAndConvert(string code, global::AspxParser.Location location)
{
SyntaxTree tree = CSharpSyntaxTree.ParseText(code, new CSharpParseOptions(kind: SourceCodeKind.Script));
var converter = new CSharpRoslynParseTreeConverter();
RootUst result = converter.Convert(new CSharpRoslynParseTree(tree)
{
SourceCodeFile = sourceCodeFile
});
result.ApplyActionToDescendants(ust => ust.TextSpan = ust.TextSpan.AddOffset(location.Start));
return(result);
}
public override Ust Visit(RootUst rootUst)
{
var newRoot = new RootUst(rootUst.SourceCodeFile, rootUst.Language)
{
SourceCodeFile = rootUst.SourceCodeFile,
Nodes = rootUst.Nodes.Select(node => Visit(node)).ToArray(),
Comments = rootUst.Comments.Select(comment => (CommentLiteral)Visit(comment)).ToArray()
};
newRoot.FillAscendants();
return(newRoot);
}
public RootUst Convert(ParseTree langParseTree)
{
var antlrParseTree = (AntlrParseTree)langParseTree;
ParserRuleContext tree = antlrParseTree.SyntaxTree;
ICharStream inputStream = tree.start.InputStream ?? tree.stop?.InputStream;
string filePath = inputStream != null ? inputStream.SourceName : "";
RootUst result = null;
if (tree != null && inputStream != null)
{
try
{
Tokens = antlrParseTree.Tokens;
root = new RootUst(langParseTree.SourceCodeFile, Language);
Ust visited = Visit(tree);
if (visited is RootUst rootUst)
{
result = rootUst;
}
else
{
result = root;
result.Node = visited;
}
result.FillAscendants();
}
catch (Exception ex) when(!(ex is ThreadAbortException))
{
Logger.LogError(new ConversionException(langParseTree.SourceCodeFile, ex));
if (result == null)
{
result = new RootUst(langParseTree.SourceCodeFile, Language)
{
Comments = ArrayUtils <CommentLiteral> .EmptyArray
};
}
}
}
else
{
result = new RootUst(langParseTree.SourceCodeFile, Language)
{
Comments = ArrayUtils <CommentLiteral> .EmptyArray
};
}
result.Comments = antlrParseTree.Comments.Select(c => new CommentLiteral(c.Text, c.GetTextSpan())).ToArray();
return(result);
}
private void RenderGraphs(RootUst rootUst, HashSet <Stage> renderStages)
{
if (renderStages.Any())
{
var renderer = new StageRenderer
{
Logger = Logger,
DumpDir = DumpDir,
Stages = RenderStages,
RenderFormat = RenderFormat,
RenderDirection = RenderDirection
};
renderer.Render(rootUst);
}
}
private Ust ParseExpressionAndConvert(string expression, int offset)
{
ExpressionSyntax node = SyntaxFactory.ParseExpression(expression);
var converter = new CSharpRoslynParseTreeConverter();
Ust result = converter.Visit(node);
RootUst resultRoot =
result as RootUst ?? new RootUst(sourceFile, Language.CSharp, result.TextSpan)
{
Node = result
};
resultRoot.SourceFile = sourceFile;
result.ApplyActionToDescendantsAndSelf(ust => ust.TextSpan = ust.TextSpan.AddOffset(offset));
return(result);
}
public void Parse_NewLine_CorrectLineColumn()
{
string fileText = File.ReadAllText(Path.Combine(TestUtility.TestsDataPath, "newLine -r-n.php"));
var lineEnds = new [] { "\r", "\n", "\r\n" };
foreach (var lineEnd in lineEnds)
{
var phpParser = new PhpAntlrParser();
string code = fileText.Replace("\r\n", lineEnd);
var sourceFile = new TextFile(code)
{
Name = "newLine.php",
};
var tokens = Language.Php.CreateAntlrLexer().GetTokens(sourceFile, out TimeSpan _);
phpParser.SourceFile = sourceFile;
var parseTree = (PhpAntlrParseTree)phpParser.Parse(tokens, out TimeSpan _);
var converter = new PhpAntlrParseTreeConverter();
RootUst ust = converter.Convert(parseTree);
Ust intNode = ust.WhereDescendantsOrSelf(
node => node is IntLiteral intLiteral && intLiteral.Value == 42).First();
LineColumnTextSpan intNodeSpan = intNode.LineColumnTextSpan;
Assert.AreEqual(1, intNodeSpan.BeginLine);
Assert.AreEqual(12, intNodeSpan.BeginColumn);
Assert.AreEqual(14, intNodeSpan.EndColumn);
Ust heredocNode = ust.WhereDescendantsOrSelf(
node => node is StringLiteral stringLiteral &&
stringLiteral.Text.StartsWith("Heredoc text")).First();
LineColumnTextSpan heredocNodeSpan = heredocNode.LineColumnTextSpan;
Assert.AreEqual(3, heredocNodeSpan.BeginLine);
Assert.AreEqual(6, heredocNodeSpan.EndLine);
Ust serverAddressNode = ust.WhereDescendantsOrSelf(
node => node is StringLiteral stringLiteral &&
stringLiteral.Text.Contains("http://127.0.0.1")).First();
LineColumnTextSpan serverAddressNodeSpan = serverAddressNode.LineColumnTextSpan;
Assert.AreEqual(8, serverAddressNodeSpan.BeginLine);
Assert.AreEqual(15, serverAddressNodeSpan.BeginColumn);
}
}
public RootUst Convert(ParseTree langParseTree)
{
var antlrParseTree = (AntlrParseTree)langParseTree;
ParserRuleContext tree = antlrParseTree.SyntaxTree;
ICharStream inputStream = tree.start.InputStream ?? tree.stop?.InputStream;
if (inputStream == null)
{
return(null);
}
RootUst result;
try
{
Tokens = Language.GetSublanguages().Length > 0 ? antlrParseTree.Tokens : new List <IToken>();
root = new RootUst(langParseTree.SourceFile, Language);
Ust visited = Visit(tree);
if (visited is RootUst rootUst)
{
result = rootUst;
}
else
{
result = root;
result.Node = visited;
}
}
catch (Exception ex) when(!(ex is ThreadAbortException))
{
Logger.LogError(new ConversionException(langParseTree.SourceFile, ex));
return(null);
}
result.Comments = antlrParseTree.Comments.Select(c => new CommentLiteral(c.Text, c.GetTextSpan())
{
Root = result
})
.ToArray();
result.FillAscendants();
return(result);
}
public RootUst Convert(ParseTree langParseTree)
{
var roslynParseTree = (CSharpRoslynParseTree)langParseTree;
SyntaxTree syntaxTree = roslynParseTree.SyntaxTree;
if (syntaxTree == null)
{
return(null);
}
RootUst result;
string filePath = syntaxTree.FilePath;
try
{
Ust visited = Visit(roslynParseTree.SyntaxTree.GetRoot());
if (visited is RootUst rootUst)
{
result = rootUst;
}
else
{
result = new RootUst(langParseTree.SourceFile, Language);
result.Node = visited;
}
result.SourceFile = langParseTree.SourceFile;
result.Comments = roslynParseTree.Comments.Select(c =>
new CommentLiteral(c.ToString(), c.GetTextSpan())
{
Root = result
})
.ToArray();
result.FillAscendants();
return(result);
}
catch (Exception ex) when(!(ex is ThreadAbortException))
{
Logger.LogError(new ConversionException(langParseTree.SourceFile, ex));
return(null);
}
}
public override Ust Visit(SyntaxNode node)
{
var children = new List <Ust>();
foreach (SyntaxNode child in node.ChildNodes())
{
var result = VisitAndReturnNullIfError(child);
if (result != null)
{
children.Add(result);
}
}
if (root == null)
{
root = new RootUst(children.FirstOrDefault()?.Root?.SourceCodeFile, CSharp.Language);
}
root.Nodes = children.ToArray();
return(root);
}
public override Ust Visit(SyntaxNode node)
{
var children = new List <Ust>();
foreach (SyntaxNode child in node.ChildNodes())
{
var result = VisitAndReturnNullIfError(child);
if (result != null)
{
children.Add(result);
}
}
if (root == null)
{
root = new RootUst(null, Language.CSharp, node.GetTextSpan());
}
root.Nodes = children.ToArray();
return(root);
}
public void Convert_Char_StringLiteralWithoutQuotes()
{
var sourceRep = new MemorySourceRepository(
@"class foo {
bar() {
obj.f1 = 'a';
obj.f2 = ""'b'"";
}
}",
"StringLiteralWithoutQuotes.java"
);
RootUst ust = null;
var workflow = new Workflow(sourceRep, stage: Stage.Ust);
workflow.UstConverted += (sender, rootUst) => ust = rootUst;
workflow.Process();
Assert.IsTrue(ust.AnyDescendantOrSelf(ustNode =>
ustNode is StringLiteral stringLiteral && stringLiteral.Text == "a"));
}
public RootUst Convert(ParseTree langParseTree)
{
try
{
var esprimaParseTree = (JavaScriptEsprimaParseTree)langParseTree;
if (SourceFile == null)
{
SourceFile = esprimaParseTree.SourceFile;
}
var program = VisitProgram(esprimaParseTree.SyntaxTree);
var rootUst = new RootUst(SourceFile, Language.JavaScript, GetTextSpan(esprimaParseTree.SyntaxTree))
{
Nodes = new Ust[] { program },
};
var comments = new Collections.List <CommentLiteral>(esprimaParseTree.Comments.Count);
foreach (Comment comment in esprimaParseTree.Comments)
{
TextSpan textSpan = GetTextSpan(comment);
comments.Add(new CommentLiteral(SourceFile.GetSubstring(textSpan), textSpan)
{
Root = rootUst,
});
}
rootUst.Comments = comments.ToArray();
rootUst.Root = ParentRoot;
rootUst.FillAscendants();
return(rootUst);
}
catch (Exception ex) when(!(ex is ThreadAbortException))
{
Logger.LogError(new ConversionException(SourceFile, ex));
return(null);
}
}
private void ProcessFile(string fileName, PatternMatcher patternMatcher, WorkflowResult workflowResult, CancellationToken cancellationToken = default)
{
RootUst rootUst = null;
try
{
Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingStarted, fileName));
rootUst = ReadParseAndConvert(fileName, workflowResult);
if (rootUst != null && Stage >= Stage.Match)
{
Stopwatch stopwatch = Stopwatch.StartNew();
var matchResults = patternMatcher.Match(rootUst);
stopwatch.Stop();
Logger.LogInfo($"File {rootUst.SourceFile.Name} matched with patterns {stopwatch.GetElapsedString()}.");
workflowResult.AddMatchTime(stopwatch.Elapsed);
foreach (MatchResult matchResult in matchResults)
{
workflowResult.ProcessMatchResult(matchResult);
}
cancellationToken.ThrowIfCancellationRequested();
RenderGraphs(rootUst, workflowResult.RenderStages);
}
}
catch (OperationCanceledException)
{
workflowResult.AddTerminatedFilesCount(1);
Logger.LogInfo($"{fileName} processing cancelled");
}
catch (ThreadAbortException)
{
workflowResult.AddTerminatedFilesCount(1);
Logger.LogInfo(new OperationCanceledException($"Processing of {fileName} terminated due to depleted timeout ({FileTimeout})"));
Thread.ResetAbort();
}
catch (Exception ex)
{
workflowResult.AddTerminatedFilesCount(1);
Logger.LogError(ex);
}
finally
{
double progress = workflowResult.TotalFilesCount == 0
? workflowResult.TotalProcessedFilesCount
: (double)workflowResult.TotalProcessedFilesCount / workflowResult.TotalFilesCount;
Logger.LogInfo(new ProgressEventArgs(progress, fileName));
Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingCompleted, fileName));
if (rootUst == null)
{
Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingIgnored, fileName));
}
cancellationToken.ThrowIfCancellationRequested();
}
}
public MatchResult(RootUst rootUst, PatternRoot pattern, IEnumerable <TextSpan> textSpans)
{
RootUst = rootUst;
Pattern = pattern;
TextSpans = textSpans.ToArray();
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null)
{
return(null);
}
JObject jObject = JObject.Load(reader);
string kind = jObject[nameof(Ust.Kind)]?.ToString() ?? "";
if (!ReflectionCache.TryGetClassType(kind, out Type type))
{
JsonUtils.LogError(Logger, JsonFile, jObject, $"Unknown UST {nameof(Ust.Kind)} {kind}");
return(null);
}
RootUst rootUst = null;
Ust ust;
if (type == typeof(RootUst))
{
string languageString = (string)jObject?[nameof(RootUst.Language)] ?? "";
Language language = !string.IsNullOrEmpty(languageString)
? languageString.ParseLanguages().FirstOrDefault()
: Uncertain.Language;
rootUst = (RootUst)Activator.CreateInstance(type, null, language);
ProcessRootUst(rootUst);
ust = rootUst;
}
else
{
ust = (Ust)Activator.CreateInstance(type);
}
if (rootAncestors.Count > 0)
{
ust.Root = rootAncestors.Peek();
}
if (ancestors.Count > 0)
{
ust.Parent = ancestors.Peek();
}
if (rootUst != null)
{
rootAncestors.Push(rootUst);
}
ancestors.Push(ust);
List <TextSpan> textSpans =
jObject[nameof(Ust.TextSpan)]?.ToTextSpans(serializer).ToList() ?? null;
if (textSpans != null && textSpans.Count > 0)
{
if (textSpans.Count == 1)
{
ust.TextSpan = textSpans[0];
}
else
{
ust.InitialTextSpans = textSpans;
ust.TextSpan = textSpans.First();
}
}
try
{
serializer.Populate(jObject.CreateReader(), ust);
}
catch (Exception ex)
{
Logger.LogError(JsonFile, jObject, ex);
}
if (!IgnoreExtraProcess)
{
ExtraProcess(ust, jObject);
}
if (rootUst != null)
{
rootAncestors.Pop();
}
ancestors.Pop();
return(ust);
}
public RootUst Simplify(RootUst ust)
{
var result = (RootUst)Simplify((Ust)ust);
return(result);
}
public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
JsonSerializer serializer)
{
if (reader.TokenType == JsonToken.Null)
{
return(null);
}
JObject jObject = JObject.Load(reader);
string kind = jObject[nameof(Ust.Kind)]?.ToString() ?? "";
if (!ReflectionCache.TryGetClassType(kind, out Type type))
{
JsonUtils.LogError(Logger, SerializedFile, jObject, $"Unknown UST {nameof(Ust.Kind)} {kind}");
return(null);
}
RootUst rootUst = null;
Ust ust;
if (type == typeof(RootUst))
{
TextFile sourceFile = jObject["SourceCodeFile"]?.ToObject <TextFile>(serializer); // Back compatibility with external serializers
string languageString = (string)jObject[nameof(RootUst.Language)] ?? "";
Enum.TryParse(languageString, out Language language);
rootUst = new RootUst(sourceFile, language);
ProcessRootUst(rootUst);
ust = rootUst;
}
else
{
ust = (Ust)Activator.CreateInstance(type);
}
if (rootAncestors.Count > 0)
{
ust.Root = rootAncestors.Peek();
}
if (ancestors.Count > 0)
{
ust.Parent = ancestors.Peek();
}
if (rootUst != null)
{
rootAncestors.Push(rootUst);
}
ancestors.Push(ust);
try
{
serializer.Populate(jObject.CreateReader(), ust);
}
catch (Exception ex)
{
Logger.LogError(SerializedFile, jObject, ex);
}
JToken textSpanToken = jObject[nameof(Ust.TextSpan)];
TextSpan[] textSpans;
if (textSpanToken is JArray textSpanArray)
{
textSpans = new TextSpan[textSpanArray.Count];
for (int i = 0; i < textSpanArray.Count; i++)
{
textSpans[i] = textSpanArray[i].ToObject <TextSpan>(serializer);
}
}
else if (textSpanToken is JToken token)
{
textSpans = new [] { token.ToObject <TextSpan>(serializer) };
}
else
{
textSpans = new TextSpan[0];
}
ust.TextSpans = textSpans;
if (!IgnoreExtraProcess)
{
ExtraProcess(ust, jObject, serializer);
}
if (rootUst != null)
{
rootAncestors.Pop();
}
ancestors.Pop();
return(ust);
}
protected virtual void ProcessRootUst(RootUst rootUst)
{
}
public MultichildExpression(TextSpan textSpan, RootUst fileNode, params Expression[] children)
: base(textSpan)
{
Expressions = children.ToList();
}
public void AddResultEntity(RootUst ust)
{
AddEntity(usts, ust);
}
