private static void GenerateData(string sourcePath, string dataPath, string csvDirectory) { Dictionary<Language, AbstractCodeParser> CodeParser = new Dictionary<Language, AbstractCodeParser>() { { Language.CPlusPlus, new CPlusPlusCodeParser() }, { Language.Java, new JavaCodeParser() }, { Language.CSharp, new CSharpCodeParser() } }; string fileLogPath = Path.Combine(dataPath, "parse.log"); string callLogPath = Path.Combine(dataPath, "methodcalls.log"); string csvPath = Path.Combine(csvDirectory, "timing.csv"); string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23))); if(!Directory.Exists(sourcePath)) { Console.Error.WriteLine("{0} does not exist", sourcePath); return; } if(File.Exists(callLogPath)) { File.Delete(callLogPath); } if(File.Exists(fileLogPath)) { File.Delete(fileLogPath); } var archive = new SrcMLArchive(dataPath); archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus; AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive); ManualResetEvent mre = new ManualResetEvent(false); Stopwatch timer = new Stopwatch(); bool startupCompleted = false; monitor.IsReadyChanged += (o, e) => { if(e.ReadyState) { timer.Stop(); startupCompleted = true; mre.Set(); } }; timer.Start(); monitor.Startup(); string[] spinner = new string[] { "\\\r", "|\r", "/\r" }; int spinner_index = -1; while(!startupCompleted) { spinner_index = (++spinner_index) % 3; Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]); startupCompleted = mre.WaitOne(5000); } timer.Stop(); Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed); Scope globalScope = null; timer.Reset(); int numberOfFailures = 0; int numberOfSuccesses = 0; int numberOfFiles = 0; Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>(); if(!File.Exists(csvPath)) { File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") }); } using(StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) { timer.Start(); foreach(var unit in archive.FileUnits) { var fileName = SrcMLElement.GetFileNameForUnit(unit); var language = SrcMLElement.GetLanguageForUnit(unit); try { var scopeForUnit = CodeParser[language].ParseFileUnit(unit); if(null == globalScope) { globalScope = scopeForUnit; } else { globalScope = globalScope.Merge(scopeForUnit); } timer.Stop(); fileLog.WriteLine("Parsing {0} PASSED", fileName); numberOfSuccesses++; } catch(Exception e) { timer.Stop(); fileLog.WriteLine("Parsing {0} FAILED", fileName); fileLog.WriteLine(e.StackTrace); var key = e.StackTrace.Split('\n')[0].Trim(); if(!errors.ContainsKey(key)) { errors[key] = new List<string>(); } errors[key].Add(fileName); numberOfFailures++; } finally { if(++numberOfFiles % 50 == 0) { Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures); csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds)); } timer.Start(); } } } timer.Stop(); Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures); Console.WriteLine("\nSummary"); Console.WriteLine("==================="); Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles); Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles); Console.WriteLine("{0} to generate data", timer.Elapsed); Console.WriteLine("See parse log at {0}", fileLogPath); OutputCallGraphByType(globalScope, jsonPath); PrintScopeReport(globalScope, sourcePath, csvDirectory); PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath); }
private void TestDataGeneration(string sourcePath, string dataPath, bool useAsyncMethods=false) { string fileLogPath = Path.Combine(dataPath, "parse.log"); string callLogPath = Path.Combine(dataPath, "methodcalls.log"); bool regenerateSrcML = shouldRegenerateSrcML; if(!Directory.Exists(sourcePath)) { Assert.Ignore("Source code for is missing"); } if(File.Exists(callLogPath)) { File.Delete(callLogPath); } if(File.Exists(fileLogPath)) { File.Delete(fileLogPath); } if(!Directory.Exists(dataPath)) { regenerateSrcML = true; } else if(shouldRegenerateSrcML) { Directory.Delete(dataPath, true); } var archive = new SrcMLArchive(dataPath, regenerateSrcML); archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus; AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive); monitor.UseAsyncMethods = useAsyncMethods; ManualResetEvent mre = new ManualResetEvent(false); DateTime start, end = DateTime.MinValue; bool startupCompleted = false; //monitor.IsReadyChanged += (o, e) => { archive.IsReadyChanged += (o, e) => { if(e.ReadyState) { end = DateTime.Now; startupCompleted = true; mre.Set(); } }; start = DateTime.Now; monitor.Startup(); startupCompleted = mre.WaitOne(120000); if(!startupCompleted) { end = DateTime.Now; } Console.WriteLine("{0} to {1} srcML", end - start, (regenerateSrcML ? "generate" : "verify")); Assert.That(startupCompleted); Scope globalScope = null; int numberOfFailures = 0; int numberOfSuccesses = 0; int numberOfFiles = 0; Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>(); start = DateTime.Now; using(var fileLog = new StreamWriter(fileLogPath)) { foreach(var unit in archive.FileUnits) { if(++numberOfFiles % 100 == 0) { Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures); } var fileName = SrcMLElement.GetFileNameForUnit(unit); var language = SrcMLElement.GetLanguageForUnit(unit); fileLog.Write("Parsing {0}", fileName); try { var scopeForUnit = CodeParser[language].ParseFileUnit(unit); if(null == globalScope) { globalScope = scopeForUnit; } else { globalScope = globalScope.Merge(scopeForUnit); } fileLog.WriteLine(" PASSED"); numberOfSuccesses++; } catch(Exception e) { fileLog.WriteLine(" FAILED"); fileLog.WriteLine(e.StackTrace); var key = e.StackTrace.Split('\n')[0].Trim(); if(!errors.ContainsKey(key)) { errors[key] = new List<string>(); } errors[key].Add(fileName); numberOfFailures++; } } } end = DateTime.Now; Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, end - start, numberOfFailures); Console.WriteLine("\nSummary"); Console.WriteLine("==================="); Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles); Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles); Console.WriteLine("{0} to generate data", end - start); Console.WriteLine(fileLogPath); PrintScopeReport(globalScope); PrintMethodCallReport(globalScope, callLogPath); PrintErrorReport(errors); monitor.Dispose(); //Assert.AreEqual(numberOfFailures, (from e in errors.Values select e.Count).Sum()); //Assert.AreEqual(0, numberOfFailures); }