public void Render(RootUst ust) { try { var renderer = new StageDotRenderer { RenderStages = Stages, IncludeHiddenTokens = IncludeHiddenTokens, RenderDirection = RenderDirection }; string dotGraph = renderer.Render(ust); string fileName = (!string.IsNullOrEmpty(Path.GetFileName(ust.SourceFile.Name)) ? ust.SourceFile.Name + "." : "") + "ust"; var graph = new GraphvizGraph(dotGraph) { OutputFormat = RenderFormat, Logger = Logger }; graph.Render(Path.Combine(DumpDir, fileName)); } catch (Exception ex) when(!(ex is ThreadAbortException)) { Logger.LogError(ex); } }
public RootUst Convert(ParseTree langParseTree) { var aspxParseTree = (AspxParseTree)langParseTree; try { RootUst result; sourceFile = langParseTree.SourceFile; Ust visited = aspxParseTree.Root.Accept(this); if (visited is RootUst rootUst) { result = rootUst; } else { result = new RootUst(sourceFile, Language); result.Node = visited; } result.FillAscendants(); return(result); } catch (Exception ex) when(!(ex is ThreadAbortException)) { Logger.LogError(new ConversionException(aspxParseTree.SourceFile, ex)); return(null); } }
public static byte[] Serialize(RootUst rootUst, bool isLineColumn, bool compress, ILogger logger, out int writeSize) { var textSpanSerializer = TextSpanSerializer.CreateWriter(); textSpanSerializer.IsLineColumn = isLineColumn; textSpanSerializer.Logger = logger; var sourceFileSerializer = FileSerializer.CreateWriter(); sourceFileSerializer.Logger = logger; var rootUstSerializer = new RootUstMessagePackSerializer { textSpanSerializer = textSpanSerializer, fileSerializer = sourceFileSerializer }; if (buffer == null) { buffer = new byte[ushort.MaxValue + 1]; } writeSize = rootUstSerializer.SerializeRootUst(ref buffer, 0, rootUst); if (compress) { var result = LZ4MessagePackSerializer.ToLZ4Binary(new ArraySegment <byte>(buffer, 0, writeSize)); return(result); } return(buffer); }
public void Convert_JavaArrayInitialization() { var sourceRep = new MemorySourceRepository( "class ArrayInitialization {\r\n" + "public void init() {\r\n" + "int[] arr1 = new int[] { 1, 2, 3 };\r\n" + "int[][] arr2 = new int[1][2];\r\n" + "int[][] arr3 = new int[1][];\r\n" + "}\r\n" + "}", "ArrayInitialization.java" ); RootUst ust = null; var workflow = new Workflow(sourceRep, stage: Stage.Ust); workflow.UstConverted += (sender, rootUst) => ust = rootUst; workflow.Process(); var intType = new TypeToken("int"); var arrayData = new List <Tuple <List <Expression>, List <Expression> > >(); // new int[] { 1, 2, 3 }; arrayData.Add(new Tuple <List <Expression>, List <Expression> >( Enumerable.Range(1, 3).Select(num => new IntLiteral(num)).ToList <Expression>(), new List <Expression> { new IntLiteral(0) } )); // new int[1][2]; arrayData.Add(new Tuple <List <Expression>, List <Expression> >( null, new List <Expression> { new IntLiteral(1), new IntLiteral(2) } )); // new int[1][]; arrayData.Add(new Tuple <List <Expression>, List <Expression> >( null, new List <Expression> { new IntLiteral(1), new IntLiteral(0) } )); for (var i = 0; i < arrayData.Count; i++) { var data = arrayData[i]; var arrayCreationExpression = new ArrayCreationExpression { Type = intType, Initializers = data.Item1, Sizes = data.Item2 }; bool exist = ust.AnyDescendantOrSelf(node => node.Equals(arrayCreationExpression)); Assert.IsTrue(exist, "Test failed on " + i + " iteration."); } }
public static byte[] Serialize(RootUst rootUst, bool isLineColumn, bool compress, ILogger logger) { byte[] result = Serialize(rootUst, isLineColumn, compress, logger, out int writeSize); if (compress) { return(result); } return(MessagePackBinary.FastCloneWithResize(result, writeSize)); }
private Ust ParseAndConvert(string code, global::AspxParser.Location location) { SyntaxTree tree = CSharpSyntaxTree.ParseText(code, new CSharpParseOptions(kind: SourceCodeKind.Script)); var converter = new CSharpRoslynParseTreeConverter(); RootUst result = converter.Convert(new CSharpRoslynParseTree(tree) { SourceCodeFile = sourceCodeFile }); result.ApplyActionToDescendants(ust => ust.TextSpan = ust.TextSpan.AddOffset(location.Start)); return(result); }
public override Ust Visit(RootUst rootUst) { var newRoot = new RootUst(rootUst.SourceCodeFile, rootUst.Language) { SourceCodeFile = rootUst.SourceCodeFile, Nodes = rootUst.Nodes.Select(node => Visit(node)).ToArray(), Comments = rootUst.Comments.Select(comment => (CommentLiteral)Visit(comment)).ToArray() }; newRoot.FillAscendants(); return(newRoot); }
public RootUst Convert(ParseTree langParseTree) { var antlrParseTree = (AntlrParseTree)langParseTree; ParserRuleContext tree = antlrParseTree.SyntaxTree; ICharStream inputStream = tree.start.InputStream ?? tree.stop?.InputStream; string filePath = inputStream != null ? inputStream.SourceName : ""; RootUst result = null; if (tree != null && inputStream != null) { try { Tokens = antlrParseTree.Tokens; root = new RootUst(langParseTree.SourceCodeFile, Language); Ust visited = Visit(tree); if (visited is RootUst rootUst) { result = rootUst; } else { result = root; result.Node = visited; } result.FillAscendants(); } catch (Exception ex) when(!(ex is ThreadAbortException)) { Logger.LogError(new ConversionException(langParseTree.SourceCodeFile, ex)); if (result == null) { result = new RootUst(langParseTree.SourceCodeFile, Language) { Comments = ArrayUtils <CommentLiteral> .EmptyArray }; } } } else { result = new RootUst(langParseTree.SourceCodeFile, Language) { Comments = ArrayUtils <CommentLiteral> .EmptyArray }; } result.Comments = antlrParseTree.Comments.Select(c => new CommentLiteral(c.Text, c.GetTextSpan())).ToArray(); return(result); }
private void RenderGraphs(RootUst rootUst, HashSet <Stage> renderStages) { if (renderStages.Any()) { var renderer = new StageRenderer { Logger = Logger, DumpDir = DumpDir, Stages = RenderStages, RenderFormat = RenderFormat, RenderDirection = RenderDirection }; renderer.Render(rootUst); } }
private Ust ParseExpressionAndConvert(string expression, int offset) { ExpressionSyntax node = SyntaxFactory.ParseExpression(expression); var converter = new CSharpRoslynParseTreeConverter(); Ust result = converter.Visit(node); RootUst resultRoot = result as RootUst ?? new RootUst(sourceFile, Language.CSharp, result.TextSpan) { Node = result }; resultRoot.SourceFile = sourceFile; result.ApplyActionToDescendantsAndSelf(ust => ust.TextSpan = ust.TextSpan.AddOffset(offset)); return(result); }
public void Parse_NewLine_CorrectLineColumn() { string fileText = File.ReadAllText(Path.Combine(TestUtility.TestsDataPath, "newLine -r-n.php")); var lineEnds = new [] { "\r", "\n", "\r\n" }; foreach (var lineEnd in lineEnds) { var phpParser = new PhpAntlrParser(); string code = fileText.Replace("\r\n", lineEnd); var sourceFile = new TextFile(code) { Name = "newLine.php", }; var tokens = Language.Php.CreateAntlrLexer().GetTokens(sourceFile, out TimeSpan _); phpParser.SourceFile = sourceFile; var parseTree = (PhpAntlrParseTree)phpParser.Parse(tokens, out TimeSpan _); var converter = new PhpAntlrParseTreeConverter(); RootUst ust = converter.Convert(parseTree); Ust intNode = ust.WhereDescendantsOrSelf( node => node is IntLiteral intLiteral && intLiteral.Value == 42).First(); LineColumnTextSpan intNodeSpan = intNode.LineColumnTextSpan; Assert.AreEqual(1, intNodeSpan.BeginLine); Assert.AreEqual(12, intNodeSpan.BeginColumn); Assert.AreEqual(14, intNodeSpan.EndColumn); Ust heredocNode = ust.WhereDescendantsOrSelf( node => node is StringLiteral stringLiteral && stringLiteral.Text.StartsWith("Heredoc text")).First(); LineColumnTextSpan heredocNodeSpan = heredocNode.LineColumnTextSpan; Assert.AreEqual(3, heredocNodeSpan.BeginLine); Assert.AreEqual(6, heredocNodeSpan.EndLine); Ust serverAddressNode = ust.WhereDescendantsOrSelf( node => node is StringLiteral stringLiteral && stringLiteral.Text.Contains("http://127.0.0.1")).First(); LineColumnTextSpan serverAddressNodeSpan = serverAddressNode.LineColumnTextSpan; Assert.AreEqual(8, serverAddressNodeSpan.BeginLine); Assert.AreEqual(15, serverAddressNodeSpan.BeginColumn); } }
public RootUst Convert(ParseTree langParseTree) { var antlrParseTree = (AntlrParseTree)langParseTree; ParserRuleContext tree = antlrParseTree.SyntaxTree; ICharStream inputStream = tree.start.InputStream ?? tree.stop?.InputStream; if (inputStream == null) { return(null); } RootUst result; try { Tokens = Language.GetSublanguages().Length > 0 ? antlrParseTree.Tokens : new List <IToken>(); root = new RootUst(langParseTree.SourceFile, Language); Ust visited = Visit(tree); if (visited is RootUst rootUst) { result = rootUst; } else { result = root; result.Node = visited; } } catch (Exception ex) when(!(ex is ThreadAbortException)) { Logger.LogError(new ConversionException(langParseTree.SourceFile, ex)); return(null); } result.Comments = antlrParseTree.Comments.Select(c => new CommentLiteral(c.Text, c.GetTextSpan()) { Root = result }) .ToArray(); result.FillAscendants(); return(result); }
public RootUst Convert(ParseTree langParseTree) { var roslynParseTree = (CSharpRoslynParseTree)langParseTree; SyntaxTree syntaxTree = roslynParseTree.SyntaxTree; if (syntaxTree == null) { return(null); } RootUst result; string filePath = syntaxTree.FilePath; try { Ust visited = Visit(roslynParseTree.SyntaxTree.GetRoot()); if (visited is RootUst rootUst) { result = rootUst; } else { result = new RootUst(langParseTree.SourceFile, Language); result.Node = visited; } result.SourceFile = langParseTree.SourceFile; result.Comments = roslynParseTree.Comments.Select(c => new CommentLiteral(c.ToString(), c.GetTextSpan()) { Root = result }) .ToArray(); result.FillAscendants(); return(result); } catch (Exception ex) when(!(ex is ThreadAbortException)) { Logger.LogError(new ConversionException(langParseTree.SourceFile, ex)); return(null); } }
public override Ust Visit(SyntaxNode node) { var children = new List <Ust>(); foreach (SyntaxNode child in node.ChildNodes()) { var result = VisitAndReturnNullIfError(child); if (result != null) { children.Add(result); } } if (root == null) { root = new RootUst(children.FirstOrDefault()?.Root?.SourceCodeFile, CSharp.Language); } root.Nodes = children.ToArray(); return(root); }
public override Ust Visit(SyntaxNode node) { var children = new List <Ust>(); foreach (SyntaxNode child in node.ChildNodes()) { var result = VisitAndReturnNullIfError(child); if (result != null) { children.Add(result); } } if (root == null) { root = new RootUst(null, Language.CSharp, node.GetTextSpan()); } root.Nodes = children.ToArray(); return(root); }
public void Convert_Char_StringLiteralWithoutQuotes() { var sourceRep = new MemorySourceRepository( @"class foo { bar() { obj.f1 = 'a'; obj.f2 = ""'b'""; } }", "StringLiteralWithoutQuotes.java" ); RootUst ust = null; var workflow = new Workflow(sourceRep, stage: Stage.Ust); workflow.UstConverted += (sender, rootUst) => ust = rootUst; workflow.Process(); Assert.IsTrue(ust.AnyDescendantOrSelf(ustNode => ustNode is StringLiteral stringLiteral && stringLiteral.Text == "a")); }
public RootUst Convert(ParseTree langParseTree) { try { var esprimaParseTree = (JavaScriptEsprimaParseTree)langParseTree; if (SourceFile == null) { SourceFile = esprimaParseTree.SourceFile; } var program = VisitProgram(esprimaParseTree.SyntaxTree); var rootUst = new RootUst(SourceFile, Language.JavaScript, GetTextSpan(esprimaParseTree.SyntaxTree)) { Nodes = new Ust[] { program }, }; var comments = new Collections.List <CommentLiteral>(esprimaParseTree.Comments.Count); foreach (Comment comment in esprimaParseTree.Comments) { TextSpan textSpan = GetTextSpan(comment); comments.Add(new CommentLiteral(SourceFile.GetSubstring(textSpan), textSpan) { Root = rootUst, }); } rootUst.Comments = comments.ToArray(); rootUst.Root = ParentRoot; rootUst.FillAscendants(); return(rootUst); } catch (Exception ex) when(!(ex is ThreadAbortException)) { Logger.LogError(new ConversionException(SourceFile, ex)); return(null); } }
private void ProcessFile(string fileName, PatternMatcher patternMatcher, WorkflowResult workflowResult, CancellationToken cancellationToken = default) { RootUst rootUst = null; try { Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingStarted, fileName)); rootUst = ReadParseAndConvert(fileName, workflowResult); if (rootUst != null && Stage >= Stage.Match) { Stopwatch stopwatch = Stopwatch.StartNew(); var matchResults = patternMatcher.Match(rootUst); stopwatch.Stop(); Logger.LogInfo($"File {rootUst.SourceFile.Name} matched with patterns {stopwatch.GetElapsedString()}."); workflowResult.AddMatchTime(stopwatch.Elapsed); foreach (MatchResult matchResult in matchResults) { workflowResult.ProcessMatchResult(matchResult); } cancellationToken.ThrowIfCancellationRequested(); RenderGraphs(rootUst, workflowResult.RenderStages); } } catch (OperationCanceledException) { workflowResult.AddTerminatedFilesCount(1); Logger.LogInfo($"{fileName} processing cancelled"); } catch (ThreadAbortException) { workflowResult.AddTerminatedFilesCount(1); Logger.LogInfo(new OperationCanceledException($"Processing of {fileName} terminated due to depleted timeout ({FileTimeout})")); Thread.ResetAbort(); } catch (Exception ex) { workflowResult.AddTerminatedFilesCount(1); Logger.LogError(ex); } finally { double progress = workflowResult.TotalFilesCount == 0 ? workflowResult.TotalProcessedFilesCount : (double)workflowResult.TotalProcessedFilesCount / workflowResult.TotalFilesCount; Logger.LogInfo(new ProgressEventArgs(progress, fileName)); Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingCompleted, fileName)); if (rootUst == null) { Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingIgnored, fileName)); } cancellationToken.ThrowIfCancellationRequested(); } }
public MatchResult(RootUst rootUst, PatternRoot pattern, IEnumerable <TextSpan> textSpans) { RootUst = rootUst; Pattern = pattern; TextSpans = textSpans.ToArray(); }
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) { if (reader.TokenType == JsonToken.Null) { return(null); } JObject jObject = JObject.Load(reader); string kind = jObject[nameof(Ust.Kind)]?.ToString() ?? ""; if (!ReflectionCache.TryGetClassType(kind, out Type type)) { JsonUtils.LogError(Logger, JsonFile, jObject, $"Unknown UST {nameof(Ust.Kind)} {kind}"); return(null); } RootUst rootUst = null; Ust ust; if (type == typeof(RootUst)) { string languageString = (string)jObject?[nameof(RootUst.Language)] ?? ""; Language language = !string.IsNullOrEmpty(languageString) ? languageString.ParseLanguages().FirstOrDefault() : Uncertain.Language; rootUst = (RootUst)Activator.CreateInstance(type, null, language); ProcessRootUst(rootUst); ust = rootUst; } else { ust = (Ust)Activator.CreateInstance(type); } if (rootAncestors.Count > 0) { ust.Root = rootAncestors.Peek(); } if (ancestors.Count > 0) { ust.Parent = ancestors.Peek(); } if (rootUst != null) { rootAncestors.Push(rootUst); } ancestors.Push(ust); List <TextSpan> textSpans = jObject[nameof(Ust.TextSpan)]?.ToTextSpans(serializer).ToList() ?? null; if (textSpans != null && textSpans.Count > 0) { if (textSpans.Count == 1) { ust.TextSpan = textSpans[0]; } else { ust.InitialTextSpans = textSpans; ust.TextSpan = textSpans.First(); } } try { serializer.Populate(jObject.CreateReader(), ust); } catch (Exception ex) { Logger.LogError(JsonFile, jObject, ex); } if (!IgnoreExtraProcess) { ExtraProcess(ust, jObject); } if (rootUst != null) { rootAncestors.Pop(); } ancestors.Pop(); return(ust); }
public RootUst Simplify(RootUst ust) { var result = (RootUst)Simplify((Ust)ust); return(result); }
public override object ReadJson(JsonReader reader, Type objectType, object existingValue, JsonSerializer serializer) { if (reader.TokenType == JsonToken.Null) { return(null); } JObject jObject = JObject.Load(reader); string kind = jObject[nameof(Ust.Kind)]?.ToString() ?? ""; if (!ReflectionCache.TryGetClassType(kind, out Type type)) { JsonUtils.LogError(Logger, SerializedFile, jObject, $"Unknown UST {nameof(Ust.Kind)} {kind}"); return(null); } RootUst rootUst = null; Ust ust; if (type == typeof(RootUst)) { TextFile sourceFile = jObject["SourceCodeFile"]?.ToObject <TextFile>(serializer); // Back compatibility with external serializers string languageString = (string)jObject[nameof(RootUst.Language)] ?? ""; Enum.TryParse(languageString, out Language language); rootUst = new RootUst(sourceFile, language); ProcessRootUst(rootUst); ust = rootUst; } else { ust = (Ust)Activator.CreateInstance(type); } if (rootAncestors.Count > 0) { ust.Root = rootAncestors.Peek(); } if (ancestors.Count > 0) { ust.Parent = ancestors.Peek(); } if (rootUst != null) { rootAncestors.Push(rootUst); } ancestors.Push(ust); try { serializer.Populate(jObject.CreateReader(), ust); } catch (Exception ex) { Logger.LogError(SerializedFile, jObject, ex); } JToken textSpanToken = jObject[nameof(Ust.TextSpan)]; TextSpan[] textSpans; if (textSpanToken is JArray textSpanArray) { textSpans = new TextSpan[textSpanArray.Count]; for (int i = 0; i < textSpanArray.Count; i++) { textSpans[i] = textSpanArray[i].ToObject <TextSpan>(serializer); } } else if (textSpanToken is JToken token) { textSpans = new [] { token.ToObject <TextSpan>(serializer) }; } else { textSpans = new TextSpan[0]; } ust.TextSpans = textSpans; if (!IgnoreExtraProcess) { ExtraProcess(ust, jObject, serializer); } if (rootUst != null) { rootAncestors.Pop(); } ancestors.Pop(); return(ust); }
protected virtual void ProcessRootUst(RootUst rootUst) { }
public MultichildExpression(TextSpan textSpan, RootUst fileNode, params Expression[] children) : base(textSpan) { Expressions = children.ToList(); }
public void AddResultEntity(RootUst ust) { AddEntity(usts, ust); }