Exemple #1
0
        public void Render(RootUst ust)
        {
            try
            {
                var renderer = new StageDotRenderer
                {
                    RenderStages        = Stages,
                    IncludeHiddenTokens = IncludeHiddenTokens,
                    RenderDirection     = RenderDirection
                };

                string dotGraph = renderer.Render(ust);

                string fileName =
                    (!string.IsNullOrEmpty(Path.GetFileName(ust.SourceFile.Name)) ? ust.SourceFile.Name + "." : "")
                    + "ust";
                var graph = new GraphvizGraph(dotGraph)
                {
                    OutputFormat = RenderFormat,
                    Logger       = Logger
                };
                graph.Render(Path.Combine(DumpDir, fileName));
            }
            catch (Exception ex) when(!(ex is ThreadAbortException))
            {
                Logger.LogError(ex);
            }
        }
Exemple #2
0
        public RootUst Convert(ParseTree langParseTree)
        {
            var aspxParseTree = (AspxParseTree)langParseTree;

            try
            {
                RootUst result;
                sourceFile = langParseTree.SourceFile;
                Ust visited = aspxParseTree.Root.Accept(this);
                if (visited is RootUst rootUst)
                {
                    result = rootUst;
                }
                else
                {
                    result      = new RootUst(sourceFile, Language);
                    result.Node = visited;
                }
                result.FillAscendants();

                return(result);
            }
            catch (Exception ex) when(!(ex is ThreadAbortException))
            {
                Logger.LogError(new ConversionException(aspxParseTree.SourceFile, ex));
                return(null);
            }
        }
Exemple #3
0
        public static byte[] Serialize(RootUst rootUst, bool isLineColumn, bool compress, ILogger logger,
                                       out int writeSize)
        {
            var textSpanSerializer = TextSpanSerializer.CreateWriter();

            textSpanSerializer.IsLineColumn = isLineColumn;
            textSpanSerializer.Logger       = logger;

            var sourceFileSerializer = FileSerializer.CreateWriter();

            sourceFileSerializer.Logger = logger;

            var rootUstSerializer = new RootUstMessagePackSerializer
            {
                textSpanSerializer = textSpanSerializer,
                fileSerializer     = sourceFileSerializer
            };

            if (buffer == null)
            {
                buffer = new byte[ushort.MaxValue + 1];
            }

            writeSize = rootUstSerializer.SerializeRootUst(ref buffer, 0, rootUst);

            if (compress)
            {
                var result = LZ4MessagePackSerializer.ToLZ4Binary(new ArraySegment <byte>(buffer, 0, writeSize));
                return(result);
            }

            return(buffer);
        }
Exemple #4
0
        public void Convert_JavaArrayInitialization()
        {
            var sourceRep = new MemorySourceRepository(
                "class ArrayInitialization {\r\n" +
                "public void init() {\r\n" +
                "int[] arr1 = new int[] { 1, 2, 3 };\r\n" +
                "int[][] arr2 = new int[1][2];\r\n" +
                "int[][] arr3 = new int[1][];\r\n" +
                "}\r\n" +
                "}",

                "ArrayInitialization.java"
                );

            RootUst ust      = null;
            var     workflow = new Workflow(sourceRep, stage: Stage.Ust);

            workflow.UstConverted += (sender, rootUst) => ust = rootUst;
            workflow.Process();

            var intType = new TypeToken("int");

            var arrayData = new List <Tuple <List <Expression>, List <Expression> > >();

            // new int[] { 1, 2, 3 };
            arrayData.Add(new Tuple <List <Expression>, List <Expression> >(
                              Enumerable.Range(1, 3).Select(num => new IntLiteral(num)).ToList <Expression>(),
                              new List <Expression> {
                new IntLiteral(0)
            }
                              ));
            // new int[1][2];
            arrayData.Add(new Tuple <List <Expression>, List <Expression> >(
                              null,
                              new List <Expression> {
                new IntLiteral(1), new IntLiteral(2)
            }
                              ));
            // new int[1][];
            arrayData.Add(new Tuple <List <Expression>, List <Expression> >(
                              null,
                              new List <Expression> {
                new IntLiteral(1), new IntLiteral(0)
            }
                              ));

            for (var i = 0; i < arrayData.Count; i++)
            {
                var data = arrayData[i];
                var arrayCreationExpression = new ArrayCreationExpression
                {
                    Type         = intType,
                    Initializers = data.Item1,
                    Sizes        = data.Item2
                };
                bool exist = ust.AnyDescendantOrSelf(node => node.Equals(arrayCreationExpression));
                Assert.IsTrue(exist, "Test failed on " + i + " iteration.");
            }
        }
Exemple #5
0
        public static byte[] Serialize(RootUst rootUst, bool isLineColumn, bool compress, ILogger logger)
        {
            byte[] result = Serialize(rootUst, isLineColumn, compress, logger, out int writeSize);

            if (compress)
            {
                return(result);
            }

            return(MessagePackBinary.FastCloneWithResize(result, writeSize));
        }
Exemple #6
0
        private Ust ParseAndConvert(string code, global::AspxParser.Location location)
        {
            SyntaxTree tree      = CSharpSyntaxTree.ParseText(code, new CSharpParseOptions(kind: SourceCodeKind.Script));
            var        converter = new CSharpRoslynParseTreeConverter();
            RootUst    result    = converter.Convert(new CSharpRoslynParseTree(tree)
            {
                SourceCodeFile = sourceCodeFile
            });

            result.ApplyActionToDescendants(ust => ust.TextSpan = ust.TextSpan.AddOffset(location.Start));
            return(result);
        }
Exemple #7
0
        public override Ust Visit(RootUst rootUst)
        {
            var newRoot = new RootUst(rootUst.SourceCodeFile, rootUst.Language)
            {
                SourceCodeFile = rootUst.SourceCodeFile,
                Nodes          = rootUst.Nodes.Select(node => Visit(node)).ToArray(),
                Comments       = rootUst.Comments.Select(comment => (CommentLiteral)Visit(comment)).ToArray()
            };

            newRoot.FillAscendants();
            return(newRoot);
        }
Exemple #8
0
        public RootUst Convert(ParseTree langParseTree)
        {
            var antlrParseTree            = (AntlrParseTree)langParseTree;
            ParserRuleContext tree        = antlrParseTree.SyntaxTree;
            ICharStream       inputStream = tree.start.InputStream ?? tree.stop?.InputStream;
            string            filePath    = inputStream != null ? inputStream.SourceName : "";
            RootUst           result      = null;

            if (tree != null && inputStream != null)
            {
                try
                {
                    Tokens = antlrParseTree.Tokens;
                    root   = new RootUst(langParseTree.SourceCodeFile, Language);
                    Ust visited = Visit(tree);
                    if (visited is RootUst rootUst)
                    {
                        result = rootUst;
                    }
                    else
                    {
                        result      = root;
                        result.Node = visited;
                    }
                    result.FillAscendants();
                }
                catch (Exception ex) when(!(ex is ThreadAbortException))
                {
                    Logger.LogError(new ConversionException(langParseTree.SourceCodeFile, ex));

                    if (result == null)
                    {
                        result = new RootUst(langParseTree.SourceCodeFile, Language)
                        {
                            Comments = ArrayUtils <CommentLiteral> .EmptyArray
                        };
                    }
                }
            }
            else
            {
                result = new RootUst(langParseTree.SourceCodeFile, Language)
                {
                    Comments = ArrayUtils <CommentLiteral> .EmptyArray
                };
            }
            result.Comments = antlrParseTree.Comments.Select(c => new CommentLiteral(c.Text, c.GetTextSpan())).ToArray();
            return(result);
        }
Exemple #9
0
 private void RenderGraphs(RootUst rootUst, HashSet <Stage> renderStages)
 {
     if (renderStages.Any())
     {
         var renderer = new StageRenderer
         {
             Logger          = Logger,
             DumpDir         = DumpDir,
             Stages          = RenderStages,
             RenderFormat    = RenderFormat,
             RenderDirection = RenderDirection
         };
         renderer.Render(rootUst);
     }
 }
Exemple #10
0
        private Ust ParseExpressionAndConvert(string expression, int offset)
        {
            ExpressionSyntax node = SyntaxFactory.ParseExpression(expression);
            var     converter     = new CSharpRoslynParseTreeConverter();
            Ust     result        = converter.Visit(node);
            RootUst resultRoot    =
                result as RootUst ?? new RootUst(sourceFile, Language.CSharp, result.TextSpan)
            {
                Node = result
            };

            resultRoot.SourceFile = sourceFile;
            result.ApplyActionToDescendantsAndSelf(ust => ust.TextSpan = ust.TextSpan.AddOffset(offset));
            return(result);
        }
Exemple #11
0
        public void Parse_NewLine_CorrectLineColumn()
        {
            string fileText = File.ReadAllText(Path.Combine(TestUtility.TestsDataPath, "newLine -r-n.php"));
            var    lineEnds = new [] { "\r", "\n", "\r\n" };

            foreach (var lineEnd in lineEnds)
            {
                var    phpParser  = new PhpAntlrParser();
                string code       = fileText.Replace("\r\n", lineEnd);
                var    sourceFile = new TextFile(code)
                {
                    Name = "newLine.php",
                };
                var tokens = Language.Php.CreateAntlrLexer().GetTokens(sourceFile, out TimeSpan _);
                phpParser.SourceFile = sourceFile;
                var     parseTree = (PhpAntlrParseTree)phpParser.Parse(tokens, out TimeSpan _);
                var     converter = new PhpAntlrParseTreeConverter();
                RootUst ust       = converter.Convert(parseTree);

                Ust intNode = ust.WhereDescendantsOrSelf(
                    node => node is IntLiteral intLiteral && intLiteral.Value == 42).First();

                LineColumnTextSpan intNodeSpan = intNode.LineColumnTextSpan;
                Assert.AreEqual(1, intNodeSpan.BeginLine);
                Assert.AreEqual(12, intNodeSpan.BeginColumn);
                Assert.AreEqual(14, intNodeSpan.EndColumn);

                Ust heredocNode = ust.WhereDescendantsOrSelf(
                    node => node is StringLiteral stringLiteral &&
                    stringLiteral.Text.StartsWith("Heredoc text")).First();

                LineColumnTextSpan heredocNodeSpan = heredocNode.LineColumnTextSpan;
                Assert.AreEqual(3, heredocNodeSpan.BeginLine);
                Assert.AreEqual(6, heredocNodeSpan.EndLine);

                Ust serverAddressNode = ust.WhereDescendantsOrSelf(
                    node => node is StringLiteral stringLiteral &&
                    stringLiteral.Text.Contains("http://127.0.0.1")).First();

                LineColumnTextSpan serverAddressNodeSpan = serverAddressNode.LineColumnTextSpan;
                Assert.AreEqual(8, serverAddressNodeSpan.BeginLine);
                Assert.AreEqual(15, serverAddressNodeSpan.BeginColumn);
            }
        }
Exemple #12
0
        public RootUst Convert(ParseTree langParseTree)
        {
            var antlrParseTree            = (AntlrParseTree)langParseTree;
            ParserRuleContext tree        = antlrParseTree.SyntaxTree;
            ICharStream       inputStream = tree.start.InputStream ?? tree.stop?.InputStream;

            if (inputStream == null)
            {
                return(null);
            }

            RootUst result;

            try
            {
                Tokens = Language.GetSublanguages().Length > 0 ? antlrParseTree.Tokens : new List <IToken>();
                root   = new RootUst(langParseTree.SourceFile, Language);
                Ust visited = Visit(tree);
                if (visited is RootUst rootUst)
                {
                    result = rootUst;
                }
                else
                {
                    result      = root;
                    result.Node = visited;
                }
            }
            catch (Exception ex) when(!(ex is ThreadAbortException))
            {
                Logger.LogError(new ConversionException(langParseTree.SourceFile, ex));
                return(null);
            }

            result.Comments = antlrParseTree.Comments.Select(c => new CommentLiteral(c.Text, c.GetTextSpan())
            {
                Root = result
            })
                              .ToArray();

            result.FillAscendants();

            return(result);
        }
        public RootUst Convert(ParseTree langParseTree)
        {
            var        roslynParseTree = (CSharpRoslynParseTree)langParseTree;
            SyntaxTree syntaxTree      = roslynParseTree.SyntaxTree;

            if (syntaxTree == null)
            {
                return(null);
            }

            RootUst result;
            string  filePath = syntaxTree.FilePath;

            try
            {
                Ust visited = Visit(roslynParseTree.SyntaxTree.GetRoot());
                if (visited is RootUst rootUst)
                {
                    result = rootUst;
                }
                else
                {
                    result      = new RootUst(langParseTree.SourceFile, Language);
                    result.Node = visited;
                }
                result.SourceFile = langParseTree.SourceFile;
                result.Comments   = roslynParseTree.Comments.Select(c =>
                                                                    new CommentLiteral(c.ToString(), c.GetTextSpan())
                {
                    Root = result
                })
                                    .ToArray();
                result.FillAscendants();

                return(result);
            }
            catch (Exception ex) when(!(ex is ThreadAbortException))
            {
                Logger.LogError(new ConversionException(langParseTree.SourceFile, ex));
                return(null);
            }
        }
Exemple #14
0
        public override Ust Visit(SyntaxNode node)
        {
            var children = new List <Ust>();

            foreach (SyntaxNode child in node.ChildNodes())
            {
                var result = VisitAndReturnNullIfError(child);
                if (result != null)
                {
                    children.Add(result);
                }
            }

            if (root == null)
            {
                root = new RootUst(children.FirstOrDefault()?.Root?.SourceCodeFile, CSharp.Language);
            }
            root.Nodes = children.ToArray();
            return(root);
        }
        public override Ust Visit(SyntaxNode node)
        {
            var children = new List <Ust>();

            foreach (SyntaxNode child in node.ChildNodes())
            {
                var result = VisitAndReturnNullIfError(child);
                if (result != null)
                {
                    children.Add(result);
                }
            }

            if (root == null)
            {
                root = new RootUst(null, Language.CSharp, node.GetTextSpan());
            }
            root.Nodes = children.ToArray();
            return(root);
        }
Exemple #16
0
        public void Convert_Char_StringLiteralWithoutQuotes()
        {
            var sourceRep = new MemorySourceRepository(
                @"class foo {
                    bar() {
                        obj.f1 = 'a';
                        obj.f2 = ""'b'"";
                    }
                }",

                "StringLiteralWithoutQuotes.java"
                );

            RootUst ust      = null;
            var     workflow = new Workflow(sourceRep, stage: Stage.Ust);

            workflow.UstConverted += (sender, rootUst) => ust = rootUst;
            workflow.Process();

            Assert.IsTrue(ust.AnyDescendantOrSelf(ustNode =>
                                                  ustNode is StringLiteral stringLiteral && stringLiteral.Text == "a"));
        }
        public RootUst Convert(ParseTree langParseTree)
        {
            try
            {
                var esprimaParseTree = (JavaScriptEsprimaParseTree)langParseTree;
                if (SourceFile == null)
                {
                    SourceFile = esprimaParseTree.SourceFile;
                }
                var program = VisitProgram(esprimaParseTree.SyntaxTree);

                var rootUst = new RootUst(SourceFile, Language.JavaScript, GetTextSpan(esprimaParseTree.SyntaxTree))
                {
                    Nodes = new Ust[] { program },
                };

                var comments = new Collections.List <CommentLiteral>(esprimaParseTree.Comments.Count);
                foreach (Comment comment in esprimaParseTree.Comments)
                {
                    TextSpan textSpan = GetTextSpan(comment);
                    comments.Add(new CommentLiteral(SourceFile.GetSubstring(textSpan), textSpan)
                    {
                        Root = rootUst,
                    });
                }

                rootUst.Comments = comments.ToArray();
                rootUst.Root     = ParentRoot;
                rootUst.FillAscendants();

                return(rootUst);
            }
            catch (Exception ex) when(!(ex is ThreadAbortException))
            {
                Logger.LogError(new ConversionException(SourceFile, ex));
                return(null);
            }
        }
Exemple #18
0
        private void ProcessFile(string fileName, PatternMatcher patternMatcher, WorkflowResult workflowResult, CancellationToken cancellationToken = default)
        {
            RootUst rootUst = null;

            try
            {
                Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingStarted, fileName));

                rootUst = ReadParseAndConvert(fileName, workflowResult);
                if (rootUst != null && Stage >= Stage.Match)
                {
                    Stopwatch stopwatch = Stopwatch.StartNew();

                    var matchResults = patternMatcher.Match(rootUst);

                    stopwatch.Stop();
                    Logger.LogInfo($"File {rootUst.SourceFile.Name} matched with patterns {stopwatch.GetElapsedString()}.");
                    workflowResult.AddMatchTime(stopwatch.Elapsed);

                    foreach (MatchResult matchResult in matchResults)
                    {
                        workflowResult.ProcessMatchResult(matchResult);
                    }

                    cancellationToken.ThrowIfCancellationRequested();

                    RenderGraphs(rootUst, workflowResult.RenderStages);
                }
            }
            catch (OperationCanceledException)
            {
                workflowResult.AddTerminatedFilesCount(1);
                Logger.LogInfo($"{fileName} processing cancelled");
            }
            catch (ThreadAbortException)
            {
                workflowResult.AddTerminatedFilesCount(1);
                Logger.LogInfo(new OperationCanceledException($"Processing of {fileName} terminated due to depleted timeout ({FileTimeout})"));
                Thread.ResetAbort();
            }
            catch (Exception ex)
            {
                workflowResult.AddTerminatedFilesCount(1);
                Logger.LogError(ex);
            }
            finally
            {
                double progress = workflowResult.TotalFilesCount == 0
                    ? workflowResult.TotalProcessedFilesCount
                    : (double)workflowResult.TotalProcessedFilesCount / workflowResult.TotalFilesCount;
                Logger.LogInfo(new ProgressEventArgs(progress, fileName));
                Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingCompleted, fileName));

                if (rootUst == null)
                {
                    Logger.LogInfo(new MessageEventArgs(MessageType.ProcessingIgnored, fileName));
                }

                cancellationToken.ThrowIfCancellationRequested();
            }
        }
Exemple #19
0
 public MatchResult(RootUst rootUst, PatternRoot pattern, IEnumerable <TextSpan> textSpans)
 {
     RootUst   = rootUst;
     Pattern   = pattern;
     TextSpans = textSpans.ToArray();
 }
        public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
                                        JsonSerializer serializer)
        {
            if (reader.TokenType == JsonToken.Null)
            {
                return(null);
            }

            JObject jObject = JObject.Load(reader);
            string  kind    = jObject[nameof(Ust.Kind)]?.ToString() ?? "";

            if (!ReflectionCache.TryGetClassType(kind, out Type type))
            {
                JsonUtils.LogError(Logger, JsonFile, jObject, $"Unknown UST {nameof(Ust.Kind)} {kind}");
                return(null);
            }

            RootUst rootUst = null;
            Ust     ust;

            if (type == typeof(RootUst))
            {
                string   languageString = (string)jObject?[nameof(RootUst.Language)] ?? "";
                Language language       = !string.IsNullOrEmpty(languageString)
                    ? languageString.ParseLanguages().FirstOrDefault()
                    : Uncertain.Language;

                rootUst = (RootUst)Activator.CreateInstance(type, null, language);
                ProcessRootUst(rootUst);

                ust = rootUst;
            }
            else
            {
                ust = (Ust)Activator.CreateInstance(type);
            }

            if (rootAncestors.Count > 0)
            {
                ust.Root = rootAncestors.Peek();
            }
            if (ancestors.Count > 0)
            {
                ust.Parent = ancestors.Peek();
            }

            if (rootUst != null)
            {
                rootAncestors.Push(rootUst);
            }
            ancestors.Push(ust);

            List <TextSpan> textSpans =
                jObject[nameof(Ust.TextSpan)]?.ToTextSpans(serializer).ToList() ?? null;

            if (textSpans != null && textSpans.Count > 0)
            {
                if (textSpans.Count == 1)
                {
                    ust.TextSpan = textSpans[0];
                }
                else
                {
                    ust.InitialTextSpans = textSpans;
                    ust.TextSpan         = textSpans.First();
                }
            }

            try
            {
                serializer.Populate(jObject.CreateReader(), ust);
            }
            catch (Exception ex)
            {
                Logger.LogError(JsonFile, jObject, ex);
            }

            if (!IgnoreExtraProcess)
            {
                ExtraProcess(ust, jObject);
            }

            if (rootUst != null)
            {
                rootAncestors.Pop();
            }
            ancestors.Pop();

            return(ust);
        }
Exemple #21
0
        public RootUst Simplify(RootUst ust)
        {
            var result = (RootUst)Simplify((Ust)ust);

            return(result);
        }
        public override object ReadJson(JsonReader reader, Type objectType, object existingValue,
                                        JsonSerializer serializer)
        {
            if (reader.TokenType == JsonToken.Null)
            {
                return(null);
            }

            JObject jObject = JObject.Load(reader);
            string  kind    = jObject[nameof(Ust.Kind)]?.ToString() ?? "";

            if (!ReflectionCache.TryGetClassType(kind, out Type type))
            {
                JsonUtils.LogError(Logger, SerializedFile, jObject, $"Unknown UST {nameof(Ust.Kind)} {kind}");
                return(null);
            }

            RootUst rootUst = null;
            Ust     ust;

            if (type == typeof(RootUst))
            {
                TextFile sourceFile     = jObject["SourceCodeFile"]?.ToObject <TextFile>(serializer); // Back compatibility with external serializers
                string   languageString = (string)jObject[nameof(RootUst.Language)] ?? "";
                Enum.TryParse(languageString, out Language language);
                rootUst = new RootUst(sourceFile, language);
                ProcessRootUst(rootUst);

                ust = rootUst;
            }
            else
            {
                ust = (Ust)Activator.CreateInstance(type);
            }

            if (rootAncestors.Count > 0)
            {
                ust.Root = rootAncestors.Peek();
            }
            if (ancestors.Count > 0)
            {
                ust.Parent = ancestors.Peek();
            }

            if (rootUst != null)
            {
                rootAncestors.Push(rootUst);
            }
            ancestors.Push(ust);

            try
            {
                serializer.Populate(jObject.CreateReader(), ust);
            }
            catch (Exception ex)
            {
                Logger.LogError(SerializedFile, jObject, ex);
            }

            JToken textSpanToken = jObject[nameof(Ust.TextSpan)];

            TextSpan[] textSpans;

            if (textSpanToken is JArray textSpanArray)
            {
                textSpans = new TextSpan[textSpanArray.Count];
                for (int i = 0; i < textSpanArray.Count; i++)
                {
                    textSpans[i] = textSpanArray[i].ToObject <TextSpan>(serializer);
                }
            }
            else if (textSpanToken is JToken token)
            {
                textSpans = new [] { token.ToObject <TextSpan>(serializer) };
            }
            else
            {
                textSpans = new TextSpan[0];
            }

            ust.TextSpans = textSpans;

            if (!IgnoreExtraProcess)
            {
                ExtraProcess(ust, jObject, serializer);
            }

            if (rootUst != null)
            {
                rootAncestors.Pop();
            }
            ancestors.Pop();

            return(ust);
        }
 protected virtual void ProcessRootUst(RootUst rootUst)
 {
 }
Exemple #24
0
 public MultichildExpression(TextSpan textSpan, RootUst fileNode, params Expression[] children)
     : base(textSpan)
 {
     Expressions = children.ToList();
 }
Exemple #25
0
 public void AddResultEntity(RootUst ust)
 {
     AddEntity(usts, ust);
 }