private void TestDataGeneration(string sourcePath, string dataPath, bool useAsyncMethods = false) { string fileLogPath = Path.Combine(dataPath, "parse.log"); string callLogPath = Path.Combine(dataPath, "methodcalls.log"); bool regenerateSrcML = shouldRegenerateSrcML; if (!Directory.Exists(sourcePath)) { Assert.Ignore("Source code for is missing"); } if (File.Exists(callLogPath)) { File.Delete(callLogPath); } if (File.Exists(fileLogPath)) { File.Delete(fileLogPath); } if (!Directory.Exists(dataPath)) { regenerateSrcML = true; } else if (shouldRegenerateSrcML) { Directory.Delete(dataPath, true); } var archive = new SrcMLArchive(dataPath, regenerateSrcML); archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus; int numberOfFailures = 0; int numberOfSuccesses = 0; int numberOfFiles = 0; Dictionary <string, List <string> > errors = new Dictionary <string, List <string> >(); using (var fileLog = new StreamWriter(fileLogPath)) { using (var monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive)) { DateTime start, end = DateTime.MinValue; bool startupCompleted = false; start = DateTime.Now; if (useAsyncMethods) { var task = monitor.UpdateArchivesAsync().ContinueWith((t) => { end = DateTime.Now; startupCompleted = true; }); task.Wait(); } else { monitor.UpdateArchives(); end = DateTime.Now; startupCompleted = true; } if (!startupCompleted) { end = DateTime.Now; } Console.WriteLine("{0} to {1} srcML", end - start, (regenerateSrcML ? "generate" : "verify")); Assert.That(startupCompleted); using (var data = new DataRepository(archive)) { start = DateTime.Now; data.FileProcessed += (o, e) => { if (e.EventType == FileEventType.FileAdded) { numberOfFiles++; numberOfSuccesses++; if (numberOfFiles % 100 == 0) { Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures); } fileLog.WriteLine("OK {0}", e.FilePath); } }; data.ErrorRaised += (o, e) => { numberOfFiles++; numberOfFailures++; if (numberOfFiles % 100 == 0) { Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures); } ParseException pe = e.Exception as ParseException; if (pe != null) { fileLog.WriteLine("ERROR {0}", pe.FileName); fileLog.WriteLine(e.Exception.InnerException.StackTrace); var key = e.Exception.InnerException.StackTrace.Split('\n')[0].Trim(); if (!errors.ContainsKey(key)) { errors[key] = new List <string>(); } int errorLineNumber = (pe.LineNumber < 1 ? 1 : pe.LineNumber); int errorColumnNumber = (pe.ColumnNumber < 1 ? 1 : pe.ColumnNumber); var errorLocation = String.Format("{0}({1},{2})", pe.FileName, errorLineNumber, errorColumnNumber); errors[key].Add(errorLocation); } }; if (useAsyncMethods) { data.InitializeDataAsync().Wait(); } else { data.InitializeData(); } end = DateTime.Now; Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, end - start, numberOfFailures); Console.WriteLine("{0} to generate data", end - start); Console.WriteLine("\nSummary"); Console.WriteLine("==================="); Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles); Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles); Console.WriteLine("{0} to generate data", end - start); Console.WriteLine(fileLogPath); IScope globalScope; Assert.That(data.TryLockGlobalScope(Timeout.Infinite, out globalScope)); try { PrintScopeReport(globalScope); PrintMethodCallReport(globalScope, callLogPath); PrintErrorReport(errors); } finally { data.ReleaseGlobalScopeLock(); } } } } }
private static void GenerateData(string sourcePath, string dataPath, string csvDirectory) { Dictionary <Language, ICodeParser> CodeParser = new Dictionary <Language, ICodeParser>() { { Language.CPlusPlus, new CPlusPlusCodeParser() }, { Language.Java, new JavaCodeParser() }, { Language.CSharp, new CSharpCodeParser() } }; string fileLogPath = Path.Combine(dataPath, "parse.log"); string callLogPath = Path.Combine(dataPath, "methodcalls.log"); string csvPath = Path.Combine(csvDirectory, "timing.csv"); string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23))); if (!Directory.Exists(sourcePath)) { Console.Error.WriteLine("{0} does not exist", sourcePath); return; } if (File.Exists(callLogPath)) { File.Delete(callLogPath); } if (File.Exists(fileLogPath)) { File.Delete(fileLogPath); } var archive = new SrcMLArchive(dataPath); archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus; AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive); ManualResetEvent mre = new ManualResetEvent(false); Stopwatch timer = new Stopwatch(); bool startupCompleted = false; monitor.UpdateArchivesCompleted += (o, e) => { timer.Stop(); startupCompleted = true; mre.Set(); }; timer.Start(); var task = monitor.UpdateArchivesAsync(); string[] spinner = new string[] { "\\\r", "|\r", "/\r" }; int spinner_index = -1; while (!startupCompleted) { spinner_index = (++spinner_index) % 3; Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]); startupCompleted = mre.WaitOne(5000); } timer.Stop(); Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed); IScope globalScope = null; timer.Reset(); int numberOfFailures = 0; int numberOfSuccesses = 0; int numberOfFiles = 0; Dictionary <string, List <string> > errors = new Dictionary <string, List <string> >(); if (!File.Exists(csvPath)) { File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") }); } using (StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) { timer.Start(); foreach (var unit in archive.FileUnits) { var fileName = SrcMLElement.GetFileNameForUnit(unit); var language = SrcMLElement.GetLanguageForUnit(unit); try { var scopeForUnit = CodeParser[language].ParseFileUnit(unit); if (null == globalScope) { globalScope = scopeForUnit; } else { globalScope = globalScope.Merge(scopeForUnit); } timer.Stop(); fileLog.WriteLine("Parsing {0} PASSED", fileName); numberOfSuccesses++; } catch (Exception e) { timer.Stop(); fileLog.WriteLine("Parsing {0} FAILED", fileName); fileLog.WriteLine(e.StackTrace); var key = e.StackTrace.Split('\n')[0].Trim(); if (!errors.ContainsKey(key)) { errors[key] = new List <string>(); } errors[key].Add(fileName); numberOfFailures++; } finally { if (++numberOfFiles % 50 == 0) { Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures); csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds)); } timer.Start(); } } } timer.Stop(); Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures); Console.WriteLine("\nSummary"); Console.WriteLine("==================="); Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles); Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles); Console.WriteLine("{0} to generate data", timer.Elapsed); Console.WriteLine("See parse log at {0}", fileLogPath); OutputCallGraphByType(globalScope, jsonPath); PrintScopeReport(globalScope, sourcePath, csvDirectory); PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath); }