public void CalculateSimpleProjectStats() { var dataProject = new DataProject<CompleteWorkingSet>("npp_6.2.3", Path.GetFullPath("..//..//..//projects//npp_6.2.3"), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); Debug.WriteLine(dataProject.Data.GetFiles().Count() + " files"); Debug.WriteLine(globalNamespace.GetDescendants<NamespaceDefinition>().Count() + " namespaces"); Debug.WriteLine(globalNamespace.GetDescendants<TypeDefinition>().Count() + " types"); Debug.WriteLine(globalNamespace.GetDescendants<MethodDefinition>().Count() + " methods"); Debug.WriteLine(""); var orderedMethodList = from method in globalNamespace.GetDescendants<MethodDefinition>() let loc = method.GetDescendants().Count() orderby loc descending select method; var top10 = orderedMethodList.Take(10); foreach (var m in top10) { Debug.WriteLine(m.GetFullName() + " - " + m.GetDescendants().Count()); } }
public void GenerateSimpleSwum() { var dataProject = new DataProject <CompleteWorkingSet>("npp_6.2.3", Path.GetFullPath("..//..//..//projects//npp_6.2.3"), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); //get srcml stuff in order NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); //initialize swum stuff splitter = new ConservativeIdSplitter(); tagger = new UnigramTagger(); posData = new PCKimmoPartOfSpeechData(); //find an example method var guiMethod = globalNamespace.GetDescendants <MethodDefinition>().Where(m => m.Name == "saveGUIParams").First(); var guiMethodXElement = DataHelpers.GetElement(dataProject.SourceArchive, guiMethod.PrimaryLocation); //generate swum for method declaration MethodContext mc = ContextBuilder.BuildMethodContext(guiMethodXElement); MethodDeclarationNode mdn = new MethodDeclarationNode("saveGUIParams", mc); BaseVerbRule rule = new BaseVerbRule(posData, tagger, splitter); rule.ConstructSwum(mdn); Console.WriteLine(mdn.ToString()); }
public void CalculateSimpleProjectStats() { var dataProject = new DataProject <CompleteWorkingSet>("npp_6.2.3", Path.GetFullPath("..//..//..//projects//npp_6.2.3"), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); Debug.WriteLine(dataProject.Data.GetFiles().Count() + " files"); Debug.WriteLine(globalNamespace.GetDescendants <NamespaceDefinition>().Count() + " namespaces"); Debug.WriteLine(globalNamespace.GetDescendants <TypeDefinition>().Count() + " types"); Debug.WriteLine(globalNamespace.GetDescendants <MethodDefinition>().Count() + " methods"); Debug.WriteLine(""); var orderedMethodList = from method in globalNamespace.GetDescendants <MethodDefinition>() let loc = method.GetDescendants().Count() orderby loc descending select method; var top10 = orderedMethodList.Take(10); foreach (var m in top10) { Debug.WriteLine(m.GetFullName() + " - " + m.GetDescendants().Count()); } }
public static IEnumerable<MethodDefinition> ExtractAllMethodsFromDirectory(string directoryPath) { var currentDirectory = Path.GetFullPath(Assembly.GetExecutingAssembly().Location); var srcmlDirectory = Path.GetFullPath(Path.Combine(currentDirectory, @"..\..\..\..\External\SrcML")); var dataProject = new DataProject<CompleteWorkingSet>( Path.GetDirectoryName(directoryPath), // name for base directory directoryPath, // path to source files to extract srcmlDirectory); // path to srcml executables dataProject.UpdateAsync().Wait(); NamespaceDefinition globalNamespace; dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace); var methods = globalNamespace.GetDescendants<MethodDefinition>(); return methods; }
public void GenerateSwumForAnyOccassion() { var dataProject = new DataProject <CompleteWorkingSet>(folderName, Path.GetFullPath(fullFilePath), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); //get srcml stuff in order NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); //find an example method var sampleMethod = globalNamespace.GetDescendants <MethodDefinition>().Where(m => m.Name == methodName).First(); foreach (var line in sampleMethod.GetDescendants()) //goes through lines (Statement class) in method { var sampleMethod_MethodCalls = line.FindExpressions <MethodCall>(); //represents all of the method calls within the method foreach (var methodCall in sampleMethod_MethodCalls) //goes through each method call { var swummedMdn = FromMethodCallToSWUM(methodCall, globalNamespace, dataProject); if (swummedMdn != null) { Console.WriteLine("VOID RETURN"); Console.WriteLine(swummedMdn.ToString()); } } //return statement if (line is ReturnStatement) { var returnMCall = line.FindExpressions <MethodCall>().First(); var swummedMdn = FromMethodCallToSWUM(returnMCall, globalNamespace, dataProject); if (swummedMdn != null) { Console.WriteLine("RETURN STMT"); Console.WriteLine(swummedMdn.ToString()); } } } }
public void GenerateSwumForAnyOccassion() { var dataProject = new DataProject <CompleteWorkingSet>("Swum.NET-master", Path.GetFullPath("..//..//..//projects//Swum.NET-master"), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); //get srcml stuff in order NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); //find an example method var sampleMethod = globalNamespace.GetDescendants <MethodDefinition>().Where(m => m.Name == "TestConstructSwum").First(); foreach (var line in sampleMethod.GetDescendants()) { var sampleMethod_MethodCalls = line.FindExpressions <MethodCall>(); foreach (var methodCall in sampleMethod_MethodCalls) { var swummedMdn = FromMethodCallToSWUM(methodCall, globalNamespace, dataProject); if (swummedMdn != null) { Console.WriteLine("VOID RETURN"); Console.WriteLine(swummedMdn.ToString()); } } //return statement if (line is ReturnStatement) { var returnMCall = line.FindExpressions <MethodCall>().First(); var swummedMdn = FromMethodCallToSWUM(returnMCall, globalNamespace, dataProject); if (swummedMdn != null) { Console.WriteLine("RETURN STMT"); Console.WriteLine(swummedMdn.ToString()); } } } }
public void TestCompleteWorkingSet_SingleCore(RealWorldTestProject testData) { CheckThatProjectExists(testData); Console.WriteLine("{0} {1} Project Summary", testData.ProjectName, testData.Version); Console.WriteLine("============================"); using (var project = new DataProject <CompleteWorkingSet>(new LimitedConcurrencyLevelTaskScheduler(1), new FileSystemFolderMonitor(testData.FullPath, testData.DataDirectory), new SrcMLGenerator("SrcML"))) { string unknownLogPath = Path.Combine(project.StoragePath, "unknown.log"); DateTime start = DateTime.Now, end; using (var unknownLog = new StreamWriter(unknownLogPath)) { project.UnknownLog = unknownLog; project.UpdateAsync().Wait(); } end = DateTime.Now; NamespaceDefinition globalScope; Assert.That(project.WorkingSet.TryObtainReadLock(5000, out globalScope)); Assert.IsNotNull(globalScope); } }
public static IEnumerable <MethodDefinition> ExtractAllMethodsFromDirectory(string directoryPath) { var currentDirectory = Path.GetFullPath(Assembly.GetExecutingAssembly().Location); var srcmlDirectory = Path.GetFullPath(Path.Combine(currentDirectory, @"..\..\..\..\External\SrcML")); var dataProject = new DataProject <CompleteWorkingSet>( Path.GetDirectoryName(directoryPath), // name for base directory directoryPath, // path to source files to extract srcmlDirectory); // path to srcml executables dataProject.UpdateAsync().Wait(); NamespaceDefinition globalNamespace; dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace); var methods = globalNamespace.GetDescendants <MethodDefinition>(); return(methods); }
public void GenerateSwumForAnyOccassion() { var dataProject = new DataProject<CompleteWorkingSet>("Swum.NET-master", Path.GetFullPath("..//..//..//projects//Swum.NET-master"), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); //get srcml stuff in order NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); //find an example method var sampleMethod = globalNamespace.GetDescendants<MethodDefinition>().Where(m => m.Name == "TestConstructSwum").First(); foreach(var line in sampleMethod.GetDescendants()) { var sampleMethod_MethodCalls = line.FindExpressions<MethodCall>(); foreach (var methodCall in sampleMethod_MethodCalls) { var swummedMdn = FromMethodCallToSWUM(methodCall, globalNamespace, dataProject); if (swummedMdn != null) { Console.WriteLine("VOID RETURN"); Console.WriteLine(swummedMdn.ToString()); } } //return statement if (line is ReturnStatement) { var returnMCall = line.FindExpressions<MethodCall>().First(); var swummedMdn = FromMethodCallToSWUM(returnMCall, globalNamespace, dataProject); if(swummedMdn != null) { Console.WriteLine("RETURN STMT"); Console.WriteLine(swummedMdn.ToString()); } } } }
public void GenerateSimpleSwum() { var dataProject = new DataProject<CompleteWorkingSet>("npp_6.2.3", Path.GetFullPath("..//..//..//projects//npp_6.2.3"), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); //get srcml stuff in order NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); //find an example method var guiMethod = globalNamespace.GetDescendants<MethodDefinition>().Where(m => m.Name == "saveGUIParams").First(); var guiMethodXElement = DataHelpers.GetElement(dataProject.SourceArchive, guiMethod.PrimaryLocation); //generate swum for method declaration MethodContext mc = ContextBuilder.BuildMethodContext(guiMethodXElement); MethodDeclarationNode mdn = new MethodDeclarationNode("saveGUIParams", mc); BaseVerbRule rule = new BaseVerbRule(posData, tagger, splitter); Console.WriteLine("InClass = " + rule.InClass(mdn)); rule.ConstructSwum(mdn); Console.WriteLine(mdn.ToString()); }
public void TestSerialization(RealWorldTestProject testData) { using (var project = new DataProject <NullWorkingSet>(testData.DataDirectory, testData.FullPath, "SrcML")) { string unknownLogPath = Path.Combine(project.StoragePath, "unknown.log"); using (var unknownLog = new StreamWriter(unknownLogPath)) { project.UnknownLog = unknownLog; project.UpdateAsync().Wait(); long count = 0; TextWriter output = StreamWriter.Synchronized(Console.Out), error = StreamWriter.Synchronized(Console.Error); long parseElapsed = 0, deserializedElapsed = 0, compareElapsed = 0; output.WriteLine("{0,-12} {1,-12} {2,-12} {3,-12}", "# Files", "Parse", "Deserialize", "Comparison"); Parallel.ForEach(project.Data.GetFiles(), (sourcePath) => { DateTime start, end; NamespaceDefinition data; NamespaceDefinition serializedData; try { start = DateTime.Now; var fileUnit = project.SourceArchive.GetXElementForSourceFile(sourcePath); data = project.Data.Generator.Parse(fileUnit); end = DateTime.Now; Interlocked.Add(ref parseElapsed, (end - start).Ticks); } catch (Exception ex) { Console.Error.WriteLine(ex); data = null; } try { start = DateTime.Now; serializedData = project.Data.GetData(sourcePath); end = DateTime.Now; Interlocked.Add(ref deserializedElapsed, (end - start).Ticks); } catch (Exception ex) { error.WriteLine(ex); serializedData = null; } Assert.IsNotNull(data); Assert.IsNotNull(serializedData); start = DateTime.Now; DataAssert.StatementsAreEqual(data, serializedData); end = DateTime.Now; Interlocked.Add(ref compareElapsed, (end - start).Ticks); if (Interlocked.Increment(ref count) % 25 == 0) { output.WriteLine("{0,12:N0} {1,12:ss\\.fff} {2,12:ss\\.fff} {3,12:ss\\.fff}", count, new TimeSpan(parseElapsed), new TimeSpan(deserializedElapsed), new TimeSpan(compareElapsed)); } }); Console.WriteLine("Project: {0} {1}", testData.ProjectName, testData.Version); Console.WriteLine("{0,-15} {1,11:N0}", "# Files", count); Console.WriteLine("{0,-15} {1:g}", "Parsing", new TimeSpan(parseElapsed)); Console.WriteLine("{0,-15} {1:g}", "Deserializing", new TimeSpan(deserializedElapsed)); Console.WriteLine("{0,-15} {1:g}", "Comparing", new TimeSpan(compareElapsed)); Console.WriteLine("{0,-15} {1:g}", "Total", new TimeSpan(parseElapsed + deserializedElapsed + compareElapsed)); } } }
public void GenerateEndingSUnit() { /*var dataProject = new DataProject<CompleteWorkingSet>("npp_6.2.3", Path.GetFullPath("..//..//..//projects//npp_6.2.3"), "..//..//..//SrcML");*/ var dataProject = new DataProject<CompleteWorkingSet>(folderName, Path.GetFullPath(fullFilePath), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); //get srcml stuff in order NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); //initialize swum stuff splitter = new ConservativeIdSplitter(); tagger = new UnigramTagger(); posData = new PCKimmoPartOfSpeechData(); //find an example method, uses global methodName variable var testMethod = globalNamespace.GetDescendants<MethodDefinition>().Where(m => m.Name == methodName).First(); var testMethodXElement = DataHelpers.GetElement(dataProject.SourceArchive, testMethod.PrimaryLocation); //generate swum for method declaration MethodContext mc = ContextBuilder.BuildMethodContext(testMethodXElement); MethodDeclarationNode mdn = new MethodDeclarationNode(methodName, mc); //Console.WriteLine(mdn.ToString()); //returns nothing since it hasn't been written var exp = testMethod.GetDescendants(); //var verb = mdn.Action.ToString(); var expResult = exp.ElementAt(exp.Count() - 1); Console.WriteLine(expResult); MethodDeclarationNode expMDN = null; if (expResult is ReturnStatement) { Console.WriteLine("return"); } else { var mCall = expResult.FindExpressions<MethodCall>().First(); expMDN = new MethodDeclarationNode(mCall.Name, mc); } //MethodDeclarationNode mdn2 = new MethodDeclarationNode(expResult.ToString(), mc); //BaseVerbRule rule = new BaseVerbRule(posData, tagger, splitter); //Console.WriteLine("InClass = " + rule.InClass(mdn)); //REQUIRED in order for the ConstructSwum method to work //rule.ConstructSwum(mdn); //rewrites mdn.ToString to a SWUM breakdown //Console.WriteLine(mdn.Action.ToString()); BaseVerbRule rule2 = new BaseVerbRule(posData, tagger, splitter); Console.WriteLine("InClass = " + rule2.InClass(expMDN)); //REQUIRED in order for the ConstructSwum method to work rule2.ConstructSwum(expMDN); //rewrites mdn.ToString to a SWUM breakdown Console.WriteLine(expMDN.Action.ToString()); //Console.WriteLine(mdn.Action.ToString()); }
public void GenerateVoidReturnSUnit() { var dataProject = new DataProject<CompleteWorkingSet>(folderName, Path.GetFullPath(fullFilePath), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); //get srcml stuff in order NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); //initialize swum stuff splitter = new ConservativeIdSplitter(); tagger = new UnigramTagger(); posData = new PCKimmoPartOfSpeechData(); //find an example method var guiMethod = globalNamespace.GetDescendants<MethodDefinition>().Where(m => m.Name == methodName).First(); var guiMethodXElement = DataHelpers.GetElement(dataProject.SourceArchive, guiMethod.PrimaryLocation); // forget that, find ALL the methods var methods = globalNamespace.GetDescendants<MethodDefinition>().Where(m => m.Name == methodName); foreach (MethodDefinition method in methods) { //Console.WriteLine(method.ToString()); var statements = method.ChildStatements; foreach (Statement statement in statements) { var expressions = statement.GetExpressions(); foreach (Expression expression in expressions) { // Skip any expression that contains an assignment if (expression.ToString().Contains(" =") || expression.ToString().Contains(" ->")) { continue; } // Print whatever's left. It should be a void return. Console.WriteLine(expression.ToString()); // *** PoS tag it *** // convert the string to 'PhraseNode' objects so we can feed them to SWUM var pn = PhraseNode.Parse(new WordNode( expression.ToString() ).ToString() ); Console.WriteLine(pn.ToString()); // construct the "rule" to break up method names into sentences BaseVerbRule thisrule = new BaseVerbRule(posData, tagger, splitter); var methodNode = new MethodDeclarationNode(expression.ToString()); thisrule.ConstructSwum(methodNode); Console.WriteLine(methodNode.ToString()); } } } }
public void GenerateSwumForAnyOccassion() { var dataProject = new DataProject<CompleteWorkingSet>(folderName, Path.GetFullPath(fullFilePath), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); //get srcml stuff in order NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); //find an example method var sampleMethod = globalNamespace.GetDescendants<MethodDefinition>().Where(m => m.Name == methodName).First(); foreach (var line in sampleMethod.GetDescendants()) //goes through lines (Statement class) in method { var sampleMethod_MethodCalls = line.FindExpressions<MethodCall>(); //represents all of the method calls within the method foreach (var methodCall in sampleMethod_MethodCalls) //goes through each method call { var swummedMdn = FromMethodCallToSWUM(methodCall, globalNamespace, dataProject); if (swummedMdn != null) { Console.WriteLine("VOID RETURN"); Console.WriteLine(swummedMdn.ToString()); } } //return statement if (line is ReturnStatement) { var returnMCall = line.FindExpressions<MethodCall>().First(); var swummedMdn = FromMethodCallToSWUM(returnMCall, globalNamespace, dataProject); if (swummedMdn != null) { Console.WriteLine("RETURN STMT"); Console.WriteLine(swummedMdn.ToString()); } } } }
public void GenerateSameActionSUnit() { // var dataProject = new DataProject<CompleteWorkingSet>("npp_6.2.3", // Path.GetFullPath("..//..//..//projects//npp_6.2.3"), // "..//..//..//SrcML"); //var dataProject = new DataProject<CompleteWorkingSet>("CodeAnalysisToolkit", // Path.GetFullPath("..//..//..//samples"), // "..//..//..//SrcML"); var dataProject = new DataProject<CompleteWorkingSet>(folderName, Path.GetFullPath(fullFilePath), "..//..//..//SrcML"); List<String> success = new List<String>(); Dictionary<SwumRule, bool> inClasses = null; string methodName = arg; bool debug = false; ///////////////// DEBUGGING // Get SrcML stuff in order dataProject.UpdateAsync().Wait(); NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(1000, out globalNamespace)); // Initialize Swum splitter = new ConservativeIdSplitter(); tagger = new UnigramTagger(); posData = new PCKimmoPartOfSpeechData(); var methodList = globalNamespace.GetDescendants<MethodDefinition>().Where(m => m.Name == methodName); MethodDefinition topMethod = null; // Check if the method was found try { topMethod = methodList.First(); } catch (System.InvalidOperationException) { Console.WriteLine("--ERROR: Method '" + methodName + "' Not Found--"); Assert.Fail("Method '" + methodName + "' Not Found"); } var guiMethodXElement = DataHelpers.GetElement(dataProject.SourceArchive, topMethod.PrimaryLocation); //generate swum for method declaration MethodContext mc = ContextBuilder.BuildMethodContext(guiMethodXElement); MethodDeclarationNode mdn = new MethodDeclarationNode(methodName, mc); BaseVerbRule rule = new BaseVerbRule(posData, tagger, splitter); Console.WriteLine("Method = \t" + methodName); Console.WriteLine("InClass = \t" + rule.InClass(mdn)); // Get the action verb from the SWUM String methodVerb = GetMethodVerb(methodName, mc); Console.WriteLine("Verb = \t\t" + methodVerb); Console.WriteLine("============================"); // Get all of the lines of code that contains the verb in any form var expr = topMethod.GetDescendants().Where(t => t.ToString().Contains(methodVerb)).ToArray(); // Iterate each line foreach (Statement t in expr) { if (debug) Console.WriteLine("Line: " + t.ToString()); // This finds any method calls in this line and finds the verb in that method - from GetCallsTo() var methods = t.FindExpressions<MethodCall>(true).Where(c => c.ToString().ToLower().Contains(methodVerb));//c.Name.Contains(methodVerb)); // Iterate through a list of the method calls in a line and find ones that contain the verb foreach (MethodCall i in methods) { if (debug) Console.WriteLine("===\n" + i.Name); MethodDeclarationNode mdnNEW = new MethodDeclarationNode(i.Name, mc); inClasses = InClassChecker(i.Name, mc); String foundVerb = GetMethodVerb(i.Name, mc); if (foundVerb.Equals("!NONE!")) { if (debug) Console.WriteLine(" Method does not contain verb"); } else { if (debug) Console.WriteLine("GetMethodVerb= " + GetMethodVerb(i.Name, mc)); success.Add(i.Name); } if (debug) Console.WriteLine("CompareSwums= " + CompareSwums(i.Name, mc)); // Debugging if (debug) { int numbtrue = 0; foreach (KeyValuePair<SwumRule, bool> entry in inClasses) { if (entry.Value) { numbtrue++; mdnNEW = new MethodDeclarationNode(i.Name, mc); entry.Key.InClass(mdnNEW); entry.Key.ConstructSwum(mdnNEW); Console.WriteLine("\t" + entry.Key.GetType().ToString() + new String(' ', 30 - entry.Key.GetType().ToString().Length) + " = " + mdnNEW.ToString()); if (mdnNEW.Action.ToPlainString().Equals(methodVerb, StringComparison.InvariantCultureIgnoreCase)) { Console.WriteLine("\tMethod contains the verb"); } else { Console.WriteLine("\tMethod does not contain the verb" + "\n"); } } } Console.WriteLine(" Inclasses = " + numbtrue); } //end if debug } // End Method Iteration } // End Line Iteration if (success.Count == 0) { Console.WriteLine("===== No Same-Action Methods Found ====="); //Assert.Fail("No Same-Action Methods found"); } else { Console.WriteLine("\n============= SUCCESSES ==============="); foreach (String i in success) { Console.WriteLine(i); } //Assert.Pass("Same-Action methods found, check Output"); } dataProject.WorkingSet.ReleaseReadLock(); }
public void TestCompleteWorkingSet_SingleCore(RealWorldTestProject testData) { CheckThatProjectExists(testData); Console.WriteLine("{0} {1} Project Summary", testData.ProjectName, testData.Version); Console.WriteLine("============================"); using(var project = new DataProject<CompleteWorkingSet>(new LimitedConcurrencyLevelTaskScheduler(1), new FileSystemFolderMonitor(testData.FullPath, testData.DataDirectory), new SrcMLGenerator("SrcML"))) { string unknownLogPath = Path.Combine(project.StoragePath, "unknown.log"); DateTime start = DateTime.Now, end; using(var unknownLog = new StreamWriter(unknownLogPath)) { project.UnknownLog = unknownLog; project.UpdateAsync().Wait(); } end = DateTime.Now; NamespaceDefinition globalScope; Assert.That(project.WorkingSet.TryObtainReadLock(5000, out globalScope)); Assert.IsNotNull(globalScope); } }
public void GenerateSameActionSUnit() { // var dataProject = new DataProject<CompleteWorkingSet>("npp_6.2.3", // Path.GetFullPath("..//..//..//projects//npp_6.2.3"), // "..//..//..//SrcML"); //var dataProject = new DataProject<CompleteWorkingSet>("CodeAnalysisToolkit", // Path.GetFullPath("..//..//..//samples"), // "..//..//..//SrcML"); var dataProject = new DataProject <CompleteWorkingSet>(folderName, Path.GetFullPath(fullFilePath), "..//..//..//SrcML"); List <String> success = new List <String>(); Dictionary <SwumRule, bool> inClasses = null; bool debug = false; ///////////////// DEBUGGING // Get SrcML stuff in order dataProject.UpdateAsync().Wait(); NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(1000, out globalNamespace)); // Initialize Swum splitter = new ConservativeIdSplitter(); tagger = new UnigramTagger(); posData = new PCKimmoPartOfSpeechData(); var methodList = globalNamespace.GetDescendants <MethodDefinition>().Where(m => m.Name == methodName); MethodDefinition topMethod = null; // Check if the method was found try { topMethod = methodList.First(); } catch (System.InvalidOperationException) { Console.WriteLine("--ERROR: Method '" + methodName + "' Not Found--"); Assert.Fail("Method '" + methodName + "' Not Found"); } var guiMethodXElement = DataHelpers.GetElement(dataProject.SourceArchive, topMethod.PrimaryLocation); //generate swum for method declaration MethodContext mc = ContextBuilder.BuildMethodContext(guiMethodXElement); MethodDeclarationNode mdn = new MethodDeclarationNode(methodName, mc); BaseVerbRule rule = new BaseVerbRule(posData, tagger, splitter); Console.WriteLine("Method = \t" + methodName); Console.WriteLine("InClass = \t" + rule.InClass(mdn)); // Get the action verb from the SWUM String methodVerb = GetMethodVerb(methodName, mc); Console.WriteLine("Verb = \t\t" + methodVerb); Console.WriteLine("============================"); // Get all of the lines of code that contains the verb in any form var expr = topMethod.GetDescendants().Where(t => t.ToString().Contains(methodVerb)).ToArray(); // Iterate each line foreach (Statement t in expr) { if (debug) { Console.WriteLine("Line: " + t.ToString()); } // This finds any method calls in this line and finds the verb in that method - from GetCallsTo() var methods = t.FindExpressions <MethodCall>(true).Where(c => c.ToString().ToLower().Contains(methodVerb));//c.Name.Contains(methodVerb)); // Iterate through a list of the method calls in a line and find ones that contain the verb foreach (MethodCall i in methods) { if (debug) { Console.WriteLine("===\n" + i.Name); } MethodDeclarationNode mdnNEW = new MethodDeclarationNode(i.Name, mc); inClasses = InClassChecker(i.Name, mc); String foundVerb = GetMethodVerb(i.Name, mc); if (foundVerb.Equals("!NONE!")) { if (debug) { Console.WriteLine(" Method does not contain verb"); } } else { if (debug) { Console.WriteLine("GetMethodVerb= " + GetMethodVerb(i.Name, mc)); } success.Add(i.Name); } if (debug) { Console.WriteLine("CompareSwums= " + CompareSwums(i.Name, mc)); } // Debugging if (debug) { int numbtrue = 0; foreach (KeyValuePair <SwumRule, bool> entry in inClasses) { if (entry.Value) { numbtrue++; mdnNEW = new MethodDeclarationNode(i.Name, mc); entry.Key.InClass(mdnNEW); entry.Key.ConstructSwum(mdnNEW); Console.WriteLine("\t" + entry.Key.GetType().ToString() + new String(' ', 30 - entry.Key.GetType().ToString().Length) + " = " + mdnNEW.ToString()); if (mdnNEW.Action.ToPlainString().Equals(methodVerb, StringComparison.InvariantCultureIgnoreCase)) { Console.WriteLine("\tMethod contains the verb"); } else { Console.WriteLine("\tMethod does not contain the verb" + "\n"); } } } Console.WriteLine(" Inclasses = " + numbtrue); } //end if debug } // End Method Iteration } // End Line Iteration if (success.Count == 0) { Console.WriteLine("===== No Same-Action Methods Found ====="); //Assert.Fail("No Same-Action Methods found"); } else { Console.WriteLine("\n============= SUCCESSES ==============="); foreach (String i in success) { Console.WriteLine(i); } //Assert.Pass("Same-Action methods found, check Output"); } dataProject.WorkingSet.ReleaseReadLock(); } // End Same-Action main method
//This method is running after we generate the class and it contains two parts: 1. generating call graph of the target project; 2. calling GoThroughMethods to check taht does each method contain SQL local invocation or not. public void run() { Console.WriteLine("Invoke method sql extractor"); string dataDir = @"TESTNAIVE_1.0"; using (var project = new DataProject<CompleteWorkingSet>(dataDir, this.LocalProj, this.SrcmlLoc)) { Console.WriteLine("============================"); string unknownLogPath = Path.Combine(project.StoragePath, "unknown.log"); DateTime start = DateTime.Now, end; Console.WriteLine("============================"); using (var unknownLog = new StreamWriter(unknownLogPath)) { project.UnknownLog = unknownLog; project.UpdateAsync().Wait(); } end = DateTime.Now; NamespaceDefinition globalNamespace; project.WorkingSet.TryObtainReadLock(5000, out globalNamespace); try { // Step 1. Build the call graph Console.WriteLine("{0,10:N0} files", project.Data.GetFiles().Count()); Console.WriteLine("{0,10:N0} namespaces", globalNamespace.GetDescendants<NamespaceDefinition>().Count()); Console.WriteLine("{0,10:N0} types", globalNamespace.GetDescendants<TypeDefinition>().Count()); Console.WriteLine("{0,10:N0} methods", globalNamespace.GetDescendants<MethodDefinition>().Count()); var methods = globalNamespace.GetDescendants<MethodDefinition>(); this.cgm = new CGManager(); cgm.BuildCallGraph(methods); GoThroughMethods(methods); Console.WriteLine("Method Analyzing Finished! Total SQLs found: " + sqlCount+ ", total methods: " + allDirectMethods.Count); } finally { project.WorkingSet.ReleaseReadLock(); } } }
public void TestCompleteWorkingSet(RealWorldTestProject testData) { CheckThatProjectExists(testData); Console.WriteLine("{0} {1} Project Summary", testData.ProjectName, testData.Version); Console.WriteLine("============================"); using (var project = new DataProject <CompleteWorkingSet>(testData.DataDirectory, testData.FullPath, "SrcML")) { string unknownLogPath = Path.Combine(project.StoragePath, "unknown.log"); DateTime start = DateTime.Now, end; using (var unknownLog = new StreamWriter(unknownLogPath)) { project.UnknownLog = unknownLog; project.UpdateAsync().Wait(); } end = DateTime.Now; Console.WriteLine("{0} to initialize complete working set", end - start); NamespaceDefinition globalNamespace; Assert.That(project.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); try { Console.WriteLine("{0,10:N0} files", project.Data.GetFiles().Count()); Console.WriteLine("{0,10:N0} namespaces", globalNamespace.GetDescendants <NamespaceDefinition>().Count()); Console.WriteLine("{0,10:N0} types", globalNamespace.GetDescendants <TypeDefinition>().Count()); Console.WriteLine("{0,10:N0} methods", globalNamespace.GetDescendants <MethodDefinition>().Count()); var methodCalls = from statement in globalNamespace.GetDescendantsAndSelf() from expression in statement.GetExpressions() from call in expression.GetDescendantsAndSelf <MethodCall>() select call; int numMethodCalls = 0, numMatchedMethodCalls = 0, numMissedMethodCalls = 0; Stopwatch sw = new Stopwatch(); TimeSpan elapsed = new TimeSpan(0), matchedElapsed = new TimeSpan(0), missedElapsed = new TimeSpan(0); using (var callLog = new StreamWriter(Path.Combine(testData.DataDirectory, "call_log.csv"), false)) { callLog.WriteLine("Location,Call Name,Successful,Time"); foreach (var call in methodCalls) { INamedEntity match = null; sw.Restart(); try { match = call.FindMatches().FirstOrDefault(); } catch (Exception e) { project.ErrorLog.WriteLine("{0}:{1}:{2}: Call Exception {3}", call.Location.SourceFileName, call.Location.StartingLineNumber, call.Location.StartingColumnNumber, e); } sw.Stop(); numMethodCalls++; if (null != match) { numMatchedMethodCalls++; matchedElapsed += sw.Elapsed; } else { numMissedMethodCalls++; missedElapsed += sw.Elapsed; } callLog.WriteLine(String.Join(",", String.Join(":", call.Location.SourceFileName, call.Location.StartingLineNumber, call.Location.StartingColumnNumber), call.Name, (match == null ? "0" : "1"), sw.ElapsedMilliseconds)); elapsed += sw.Elapsed; } } Console.WriteLine("{0,10:N0} method calls", numMethodCalls); Console.WriteLine("{0,10:P2} of method calls matched", (float)numMatchedMethodCalls / numMethodCalls); Console.WriteLine("{0,10:N2} matches / millisecond ({1,7:N0} ms elapsed)", ((float)numMethodCalls) / elapsed.TotalMilliseconds, elapsed.TotalMilliseconds); Console.WriteLine("{0,7:N3} ms / match", (float)matchedElapsed.TotalMilliseconds / numMatchedMethodCalls); Console.WriteLine("{0,7:N3} ms / miss", (float)missedElapsed.TotalMilliseconds / numMissedMethodCalls); } finally { project.WorkingSet.ReleaseReadLock(); } } }
public void TestCompleteWorkingSet(RealWorldTestProject testData) { CheckThatProjectExists(testData); Console.WriteLine("{0} {1} Project Summary", testData.ProjectName, testData.Version); Console.WriteLine("============================"); using(var project = new DataProject<CompleteWorkingSet>(testData.DataDirectory, testData.FullPath, "SrcML")) { string unknownLogPath = Path.Combine(project.StoragePath, "unknown.log"); DateTime start = DateTime.Now, end; using(var unknownLog = new StreamWriter(unknownLogPath)) { project.UnknownLog = unknownLog; project.UpdateAsync().Wait(); } end = DateTime.Now; Console.WriteLine("{0} to initialize complete working set", end - start); NamespaceDefinition globalNamespace; Assert.That(project.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); try { Console.WriteLine("{0,10:N0} files", project.Data.GetFiles().Count()); Console.WriteLine("{0,10:N0} namespaces", globalNamespace.GetDescendants<NamespaceDefinition>().Count()); Console.WriteLine("{0,10:N0} types", globalNamespace.GetDescendants<TypeDefinition>().Count()); Console.WriteLine("{0,10:N0} methods", globalNamespace.GetDescendants<MethodDefinition>().Count()); var methodCalls = from statement in globalNamespace.GetDescendantsAndSelf() from expression in statement.GetExpressions() from call in expression.GetDescendantsAndSelf<MethodCall>() select call; int numMethodCalls = 0, numMatchedMethodCalls = 0, numMissedMethodCalls = 0; Stopwatch sw = new Stopwatch(); TimeSpan elapsed = new TimeSpan(0), matchedElapsed = new TimeSpan(0), missedElapsed = new TimeSpan(0); using(var callLog = new StreamWriter(Path.Combine(testData.DataDirectory, "call_log.csv"), false)) { callLog.WriteLine("Location,Call Name,Successful,Time"); foreach(var call in methodCalls) { INamedEntity match = null; sw.Restart(); try { match = call.FindMatches().FirstOrDefault(); } catch(Exception e) { project.ErrorLog.WriteLine("{0}:{1}:{2}: Call Exception {3}", call.Location.SourceFileName, call.Location.StartingLineNumber, call.Location.StartingColumnNumber, e); } sw.Stop(); numMethodCalls++; if(null != match) { numMatchedMethodCalls++; matchedElapsed += sw.Elapsed; } else { numMissedMethodCalls++; missedElapsed += sw.Elapsed; } callLog.WriteLine(String.Join(",", String.Join(":", call.Location.SourceFileName, call.Location.StartingLineNumber, call.Location.StartingColumnNumber), call.Name, (match == null ? "0" : "1"), sw.ElapsedMilliseconds)); elapsed += sw.Elapsed; } } Console.WriteLine("{0,10:N0} method calls", numMethodCalls); Console.WriteLine("{0,10:P2} of method calls matched", (float) numMatchedMethodCalls / numMethodCalls); Console.WriteLine("{0,10:N2} matches / millisecond ({1,7:N0} ms elapsed)", ((float) numMethodCalls) / elapsed.TotalMilliseconds, elapsed.TotalMilliseconds); Console.WriteLine("{0,7:N3} ms / match", (float) matchedElapsed.TotalMilliseconds / numMatchedMethodCalls); Console.WriteLine("{0,7:N3} ms / miss", (float) missedElapsed.TotalMilliseconds / numMissedMethodCalls); } finally { project.WorkingSet.ReleaseReadLock(); } } }
public void GenerateEndingSUnit() { /*var dataProject = new DataProject<CompleteWorkingSet>("npp_6.2.3", * Path.GetFullPath("..//..//..//projects//npp_6.2.3"), * "..//..//..//SrcML");*/ var dataProject = new DataProject <CompleteWorkingSet>(folderName, Path.GetFullPath(fullFilePath), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); //get srcml stuff in order NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); //initialize swum stuff splitter = new ConservativeIdSplitter(); tagger = new UnigramTagger(); posData = new PCKimmoPartOfSpeechData(); //find an example method, uses global methodName variable var testMethod = globalNamespace.GetDescendants <MethodDefinition>().Where(m => m.Name == methodName).First(); var testMethodXElement = DataHelpers.GetElement(dataProject.SourceArchive, testMethod.PrimaryLocation); //generate swum for method declaration MethodContext mc = ContextBuilder.BuildMethodContext(testMethodXElement); MethodDeclarationNode mdn = new MethodDeclarationNode(methodName, mc); //Console.WriteLine(mdn.ToString()); //returns nothing since it hasn't been written var exp = testMethod.GetDescendants(); //var verb = mdn.Action.ToString(); var expResult = exp.ElementAt(exp.Count() - 1); Console.WriteLine(expResult); MethodDeclarationNode expMDN = null; if (expResult is ReturnStatement) { Console.WriteLine("return"); } else if (expResult.ToString().Contains(" = ")) { Console.WriteLine("Ending S Unit contains '='"); } else { var mCall = expResult.FindExpressions <MethodCall>().First(); expMDN = new MethodDeclarationNode(mCall.Name, mc); } //MethodDeclarationNode mdn2 = new MethodDeclarationNode(expResult.ToString(), mc); //BaseVerbRule rule = new BaseVerbRule(posData, tagger, splitter); //Console.WriteLine("InClass = " + rule.InClass(mdn)); //REQUIRED in order for the ConstructSwum method to work //rule.ConstructSwum(mdn); //rewrites mdn.ToString to a SWUM breakdown //Console.WriteLine(mdn.Action.ToString()); BaseVerbRule rule2 = new BaseVerbRule(posData, tagger, splitter); Console.WriteLine("InClass = " + rule2.InClass(expMDN)); //REQUIRED in order for the ConstructSwum method to work rule2.ConstructSwum(expMDN); //rewrites mdn.ToString to a SWUM breakdown Console.WriteLine(expMDN.Action.ToString()); //Console.WriteLine(mdn.Action.ToString()); }
public void GenerateVoidReturnSUnit() { var dataProject = new DataProject <CompleteWorkingSet>(folderName, Path.GetFullPath(fullFilePath), "..//..//..//SrcML"); dataProject.UpdateAsync().Wait(); //get srcml stuff in order NamespaceDefinition globalNamespace; Assert.That(dataProject.WorkingSet.TryObtainReadLock(5000, out globalNamespace)); //initialize swum stuff splitter = new ConservativeIdSplitter(); tagger = new UnigramTagger(); posData = new PCKimmoPartOfSpeechData(); //find an example method var guiMethod = globalNamespace.GetDescendants <MethodDefinition>().Where(m => m.Name == methodName).First(); var guiMethodXElement = DataHelpers.GetElement(dataProject.SourceArchive, guiMethod.PrimaryLocation); // forget that, find ALL the methods var methods = globalNamespace.GetDescendants <MethodDefinition>().Where(m => m.Name == methodName); foreach (MethodDefinition method in methods) { //Console.WriteLine(method.ToString()); var statements = method.ChildStatements; foreach (Statement statement in statements) { var expressions = statement.GetExpressions(); foreach (Expression expression in expressions) { // Skip any expression that contains an assignment if (expression.ToString().Contains(" =") || expression.ToString().Contains(" ->")) { continue; } // Print whatever's left. It should be a void return. Console.WriteLine(expression.ToString()); // *** PoS tag it *** // convert the string to 'PhraseNode' objects so we can feed them to SWUM var pn = PhraseNode.Parse(new WordNode(expression.ToString()).ToString()); Console.WriteLine(pn.ToString()); // construct the "rule" to break up method names into sentences BaseVerbRule thisrule = new BaseVerbRule(posData, tagger, splitter); var methodNode = new MethodDeclarationNode(expression.ToString()); thisrule.ConstructSwum(methodNode); Console.WriteLine(methodNode.ToString()); } } } }
public void run() { Console.Out.WriteLine("Invoke call graph generator "); string dataDir = @"TESTNAIVE_1.0"; //string proPath = @"C:\Users\[email protected]\Documents\RunningTest\Input\ConsoleApplication1"; //string proPath = @"C:\Users\[email protected]\Documents\RunningTest\Input\SrcML\ABB.SrcML"; using (var project = new DataProject <CompleteWorkingSet>(dataDir, this.LocalProj, this.SrcmlLoc)) { Console.WriteLine("============================"); string unknownLogPath = Path.Combine(project.StoragePath, "unknown.log"); DateTime start = DateTime.Now, end; Console.WriteLine("============================"); using (var unknownLog = new StreamWriter(unknownLogPath)) { project.UnknownLog = unknownLog; project.UpdateAsync().Wait(); } end = DateTime.Now; NamespaceDefinition globalNamespace; project.WorkingSet.TryObtainReadLock(5000, out globalNamespace); try { // Step 1. Build the call graph Console.WriteLine("{0,10:N0} files", project.Data.GetFiles().Count()); Console.WriteLine("{0,10:N0} namespaces", globalNamespace.GetDescendants <NamespaceDefinition>().Count()); Console.WriteLine("{0,10:N0} types", globalNamespace.GetDescendants <TypeDefinition>().Count()); Console.WriteLine("{0,10:N0} methods", globalNamespace.GetDescendants <MethodDefinition>().Count()); var methods = globalNamespace.GetDescendants <MethodDefinition>(); CGManager cgm = new CGManager(); cgm.BuildCallGraph(methods); // Step 2. Testing Console.WriteLine("====== test 1 ========= "); foreach (MethodDefinition m in methods) { Console.WriteLine("Method Name : {0}", m.GetFullName()); List <List <MethodDefinition> > paths = cgm.FindCalleeList(m); foreach (List <MethodDefinition> path in paths) { foreach (MethodDefinition mc in path) { Console.Write("{0}--->", mc.Name); } Console.WriteLine(""); } } Console.WriteLine("====== test 2 ========= "); List <List <MethodDefinition> > paths2 = cgm.FindCalleeListByName("ConsoleApplication1.Class1.m1"); foreach (List <MethodDefinition> path in paths2) { foreach (MethodDefinition mc in path) { Console.Write("{0}--->", mc.Name); } Console.WriteLine(""); } Console.WriteLine("====== test 3 ========= "); int sum = 0; int pathNum = 0; foreach (MethodDefinition m in methods) { Console.WriteLine("Method Name : {0}", m.GetFullName()); List <List <MethodDefinition> > paths = cgm.FindCallerList(m); foreach (List <MethodDefinition> path in paths) { sum += path.Count; pathNum++; foreach (MethodDefinition mc in path) { Console.Write("{0}<---", mc.Name); } Console.WriteLine(""); } } Console.WriteLine("average level : " + (double)sum / pathNum); } finally { project.WorkingSet.ReleaseReadLock(); } } Console.ReadLine(); }
/// <summary> /// Step 2 of DBScribeHibernate. /// Build call graph for the target project. /// Also, topologically sorted the methods in the call graph, for later bottom-up method description propagation. /// </summary> public void Step2_1_GenerateCallGraph() { // Get methods from SrcML.net //Console.Out.WriteLine("Invoke call graph generator "); string dataDir = @"TESTNAIVE_1.0"; string localProj = TargetProjPath + "\\" + ProjName; using (var project = new DataProject<CompleteWorkingSet>(dataDir, localProj, Constants.SrcmlLoc)) { string unknownLogPath = Path.Combine(project.StoragePath, "unknown.log"); DateTime start = DateTime.Now, end; using (var unknownLog = new StreamWriter(unknownLogPath)) { project.UnknownLog = unknownLog; project.UpdateAsync().Wait(); } end = DateTime.Now; project.WorkingSet.TryObtainReadLock(5000, out globalNamespace); try { // return IEnumerable<MethodDefinition> type methods = globalNamespace.GetDescendants<MethodDefinition>(); num_of_methods = globalNamespace.GetDescendants<MethodDefinition>().Count(); Console.WriteLine("# of methods = " + num_of_methods); cgm = new CGManager(); cgm.BuildCallGraph(methods); bottomUpSortedMethods = InvokeCGManager.GetBottomUpSortedMethodsFromCallGraph(methods, cgm); } finally { project.WorkingSet.ReleaseReadLock(); } } }
public void TestSerialization(RealWorldTestProject testData) { using(var project = new DataProject<NullWorkingSet>(testData.DataDirectory, testData.FullPath, "SrcML")) { string unknownLogPath = Path.Combine(project.StoragePath, "unknown.log"); using(var unknownLog = new StreamWriter(unknownLogPath)) { project.UnknownLog = unknownLog; project.UpdateAsync().Wait(); long count = 0; TextWriter output = StreamWriter.Synchronized(Console.Out), error = StreamWriter.Synchronized(Console.Error); long parseElapsed = 0, deserializedElapsed = 0, compareElapsed = 0; output.WriteLine("{0,-12} {1,-12} {2,-12} {3,-12}", "# Files", "Parse", "Deserialize", "Comparison"); Parallel.ForEach(project.Data.GetFiles(), (sourcePath) => { DateTime start, end; NamespaceDefinition data; NamespaceDefinition serializedData; try { start = DateTime.Now; var fileUnit = project.SourceArchive.GetXElementForSourceFile(sourcePath); data = project.Data.Generator.Parse(fileUnit); end = DateTime.Now; Interlocked.Add(ref parseElapsed, (end - start).Ticks); } catch(Exception ex) { Console.Error.WriteLine(ex); data = null; } try { start = DateTime.Now; serializedData = project.Data.GetData(sourcePath); end = DateTime.Now; Interlocked.Add(ref deserializedElapsed, (end - start).Ticks); } catch(Exception ex) { error.WriteLine(ex); serializedData = null; } Assert.IsNotNull(data); Assert.IsNotNull(serializedData); start = DateTime.Now; DataAssert.StatementsAreEqual(data, serializedData); end = DateTime.Now; Interlocked.Add(ref compareElapsed, (end - start).Ticks); if(Interlocked.Increment(ref count) % 25 == 0) { output.WriteLine("{0,12:N0} {1,12:ss\\.fff} {2,12:ss\\.fff} {3,12:ss\\.fff}", count, new TimeSpan(parseElapsed), new TimeSpan(deserializedElapsed), new TimeSpan(compareElapsed)); } }); Console.WriteLine("Project: {0} {1}", testData.ProjectName, testData.Version); Console.WriteLine("{0,-15} {1,11:N0}", "# Files", count); Console.WriteLine("{0,-15} {1:g}", "Parsing", new TimeSpan(parseElapsed)); Console.WriteLine("{0,-15} {1:g}", "Deserializing", new TimeSpan(deserializedElapsed)); Console.WriteLine("{0,-15} {1:g}", "Comparing", new TimeSpan(compareElapsed)); Console.WriteLine("{0,-15} {1:g}", "Total", new TimeSpan(parseElapsed + deserializedElapsed + compareElapsed)); } } }