public MainWindow() { var archivePath = FSPath.Combine(".", "ARCHIVE"); _monitor = new FileSystemFolderMonitor(".", archivePath, new LastModifiedArchive(archivePath), new SrcMLArchive(archivePath)); _monitor.FileChanged += _archive_SourceFileChanged; _monitor.StartMonitoring(); InitializeComponent(); }
private static void GenerateData(string sourcePath, string dataPath, string csvDirectory) { Dictionary<Language, AbstractCodeParser> CodeParser = new Dictionary<Language, AbstractCodeParser>() { { Language.CPlusPlus, new CPlusPlusCodeParser() }, { Language.Java, new JavaCodeParser() }, { Language.CSharp, new CSharpCodeParser() } }; string fileLogPath = Path.Combine(dataPath, "parse.log"); string callLogPath = Path.Combine(dataPath, "methodcalls.log"); string csvPath = Path.Combine(csvDirectory, "timing.csv"); string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23))); if(!Directory.Exists(sourcePath)) { Console.Error.WriteLine("{0} does not exist", sourcePath); return; } if(File.Exists(callLogPath)) { File.Delete(callLogPath); } if(File.Exists(fileLogPath)) { File.Delete(fileLogPath); } var archive = new SrcMLArchive(dataPath); archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus; AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive); ManualResetEvent mre = new ManualResetEvent(false); Stopwatch timer = new Stopwatch(); bool startupCompleted = false; monitor.IsReadyChanged += (o, e) => { if(e.ReadyState) { timer.Stop(); startupCompleted = true; mre.Set(); } }; timer.Start(); monitor.Startup(); string[] spinner = new string[] { "\\\r", "|\r", "/\r" }; int spinner_index = -1; while(!startupCompleted) { spinner_index = (++spinner_index) % 3; Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]); startupCompleted = mre.WaitOne(5000); } timer.Stop(); Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed); Scope globalScope = null; timer.Reset(); int numberOfFailures = 0; int numberOfSuccesses = 0; int numberOfFiles = 0; Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>(); if(!File.Exists(csvPath)) { File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") }); } using(StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) { timer.Start(); foreach(var unit in archive.FileUnits) { var fileName = SrcMLElement.GetFileNameForUnit(unit); var language = SrcMLElement.GetLanguageForUnit(unit); try { var scopeForUnit = CodeParser[language].ParseFileUnit(unit); if(null == globalScope) { globalScope = scopeForUnit; } else { globalScope = globalScope.Merge(scopeForUnit); } timer.Stop(); fileLog.WriteLine("Parsing {0} PASSED", fileName); numberOfSuccesses++; } catch(Exception e) { timer.Stop(); fileLog.WriteLine("Parsing {0} FAILED", fileName); fileLog.WriteLine(e.StackTrace); var key = e.StackTrace.Split('\n')[0].Trim(); if(!errors.ContainsKey(key)) { errors[key] = new List<string>(); } errors[key].Add(fileName); numberOfFailures++; } finally { if(++numberOfFiles % 50 == 0) { Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures); csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds)); } timer.Start(); } } } timer.Stop(); Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures); Console.WriteLine("\nSummary"); Console.WriteLine("==================="); Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles); Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles); Console.WriteLine("{0} to generate data", timer.Elapsed); Console.WriteLine("See parse log at {0}", fileLogPath); OutputCallGraphByType(globalScope, jsonPath); PrintScopeReport(globalScope, sourcePath, csvDirectory); PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath); }
private void TestDataGeneration(string sourcePath, string dataPath, bool useAsyncMethods = false) { string fileLogPath = Path.Combine(dataPath, "parse.log"); string callLogPath = Path.Combine(dataPath, "methodcalls.log"); bool regenerateSrcML = shouldRegenerateSrcML; if (!Directory.Exists(sourcePath)) { Assert.Ignore("Source code for is missing"); } if (File.Exists(callLogPath)) { File.Delete(callLogPath); } if (File.Exists(fileLogPath)) { File.Delete(fileLogPath); } if (!Directory.Exists(dataPath)) { regenerateSrcML = true; } else if (shouldRegenerateSrcML) { Directory.Delete(dataPath, true); } var archive = new SrcMLArchive(dataPath, regenerateSrcML); archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus; int numberOfFailures = 0; int numberOfSuccesses = 0; int numberOfFiles = 0; Dictionary <string, List <string> > errors = new Dictionary <string, List <string> >(); using (var fileLog = new StreamWriter(fileLogPath)) { using (var monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive)) { DateTime start, end = DateTime.MinValue; bool startupCompleted = false; start = DateTime.Now; if (useAsyncMethods) { var task = monitor.UpdateArchivesAsync().ContinueWith((t) => { end = DateTime.Now; startupCompleted = true; }); task.Wait(); } else { monitor.UpdateArchives(); end = DateTime.Now; startupCompleted = true; } if (!startupCompleted) { end = DateTime.Now; } Console.WriteLine("{0} to {1} srcML", end - start, (regenerateSrcML ? "generate" : "verify")); Assert.That(startupCompleted); using (var data = new DataRepository(archive)) { start = DateTime.Now; data.FileProcessed += (o, e) => { if (e.EventType == FileEventType.FileAdded) { numberOfFiles++; numberOfSuccesses++; if (numberOfFiles % 100 == 0) { Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures); } fileLog.WriteLine("OK {0}", e.FilePath); } }; data.ErrorRaised += (o, e) => { numberOfFiles++; numberOfFailures++; if (numberOfFiles % 100 == 0) { Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures); } ParseException pe = e.Exception as ParseException; if (pe != null) { fileLog.WriteLine("ERROR {0}", pe.FileName); fileLog.WriteLine(e.Exception.InnerException.StackTrace); var key = e.Exception.InnerException.StackTrace.Split('\n')[0].Trim(); if (!errors.ContainsKey(key)) { errors[key] = new List <string>(); } int errorLineNumber = (pe.LineNumber < 1 ? 1 : pe.LineNumber); int errorColumnNumber = (pe.ColumnNumber < 1 ? 1 : pe.ColumnNumber); var errorLocation = String.Format("{0}({1},{2})", pe.FileName, errorLineNumber, errorColumnNumber); errors[key].Add(errorLocation); } }; if (useAsyncMethods) { data.InitializeDataAsync().Wait(); } else { data.InitializeData(); } end = DateTime.Now; Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, end - start, numberOfFailures); Console.WriteLine("{0} to generate data", end - start); Console.WriteLine("\nSummary"); Console.WriteLine("==================="); Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles); Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles); Console.WriteLine("{0} to generate data", end - start); Console.WriteLine(fileLogPath); IScope globalScope; Assert.That(data.TryLockGlobalScope(Timeout.Infinite, out globalScope)); try { PrintScopeReport(globalScope); PrintMethodCallReport(globalScope, callLogPath); PrintErrorReport(errors); } finally { data.ReleaseGlobalScopeLock(); } } } } }
private static void GenerateData(string sourcePath, string dataPath, string csvDirectory) { Dictionary <Language, ICodeParser> CodeParser = new Dictionary <Language, ICodeParser>() { { Language.CPlusPlus, new CPlusPlusCodeParser() }, { Language.Java, new JavaCodeParser() }, { Language.CSharp, new CSharpCodeParser() } }; string fileLogPath = Path.Combine(dataPath, "parse.log"); string callLogPath = Path.Combine(dataPath, "methodcalls.log"); string csvPath = Path.Combine(csvDirectory, "timing.csv"); string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23))); if (!Directory.Exists(sourcePath)) { Console.Error.WriteLine("{0} does not exist", sourcePath); return; } if (File.Exists(callLogPath)) { File.Delete(callLogPath); } if (File.Exists(fileLogPath)) { File.Delete(fileLogPath); } var archive = new SrcMLArchive(dataPath); archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus; AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive); ManualResetEvent mre = new ManualResetEvent(false); Stopwatch timer = new Stopwatch(); bool startupCompleted = false; monitor.UpdateArchivesCompleted += (o, e) => { timer.Stop(); startupCompleted = true; mre.Set(); }; timer.Start(); var task = monitor.UpdateArchivesAsync(); string[] spinner = new string[] { "\\\r", "|\r", "/\r" }; int spinner_index = -1; while (!startupCompleted) { spinner_index = (++spinner_index) % 3; Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]); startupCompleted = mre.WaitOne(5000); } timer.Stop(); Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed); IScope globalScope = null; timer.Reset(); int numberOfFailures = 0; int numberOfSuccesses = 0; int numberOfFiles = 0; Dictionary <string, List <string> > errors = new Dictionary <string, List <string> >(); if (!File.Exists(csvPath)) { File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") }); } using (StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) { timer.Start(); foreach (var unit in archive.FileUnits) { var fileName = SrcMLElement.GetFileNameForUnit(unit); var language = SrcMLElement.GetLanguageForUnit(unit); try { var scopeForUnit = CodeParser[language].ParseFileUnit(unit); if (null == globalScope) { globalScope = scopeForUnit; } else { globalScope = globalScope.Merge(scopeForUnit); } timer.Stop(); fileLog.WriteLine("Parsing {0} PASSED", fileName); numberOfSuccesses++; } catch (Exception e) { timer.Stop(); fileLog.WriteLine("Parsing {0} FAILED", fileName); fileLog.WriteLine(e.StackTrace); var key = e.StackTrace.Split('\n')[0].Trim(); if (!errors.ContainsKey(key)) { errors[key] = new List <string>(); } errors[key].Add(fileName); numberOfFailures++; } finally { if (++numberOfFiles % 50 == 0) { Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures); csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds)); } timer.Start(); } } } timer.Stop(); Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures); Console.WriteLine("\nSummary"); Console.WriteLine("==================="); Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles); Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles); Console.WriteLine("{0} to generate data", timer.Elapsed); Console.WriteLine("See parse log at {0}", fileLogPath); OutputCallGraphByType(globalScope, jsonPath); PrintScopeReport(globalScope, sourcePath, csvDirectory); PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath); }
private void TestDataGeneration_Concurrent(string sourcePath, string dataPath) { string fileLogPath = Path.Combine(dataPath, "parse.log"); string callLogPath = Path.Combine(dataPath, "methodcalls.log"); bool regenerateSrcML = shouldRegenerateSrcML; if(!Directory.Exists(sourcePath)) { Assert.Ignore("Source code for is missing"); } if(File.Exists(callLogPath)) { File.Delete(callLogPath); } if(File.Exists(fileLogPath)) { File.Delete(fileLogPath); } if(!Directory.Exists(dataPath)) { regenerateSrcML = true; } else if(shouldRegenerateSrcML) { Directory.Delete(dataPath, true); } var archive = new SrcMLArchive(dataPath, regenerateSrcML); archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus; AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive); ManualResetEvent mre = new ManualResetEvent(false); Stopwatch sw = new Stopwatch(); bool startupCompleted = false; monitor.IsReadyChanged += (o, e) => { if(e.ReadyState) { sw.Stop(); startupCompleted = true; mre.Set(); } }; sw.Start(); //add the concurrency test: ZL monitor.Startup_Concurrent(); startupCompleted = mre.WaitOne(60000); if(sw.IsRunning) { sw.Stop(); } Console.WriteLine("{0} to {1} srcML", sw.Elapsed, (regenerateSrcML ? "generate" : "verify")); Assert.That(startupCompleted); Scope globalScope = null; sw.Reset(); int numberOfFailures = 0; int numberOfSuccesses = 0; int numberOfFiles = 0; Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>(); sw.Start(); using(var fileLog = new StreamWriter(fileLogPath)) { using(BlockingCollection<NamespaceDefinition> bc = new BlockingCollection<NamespaceDefinition>()) { Task.Factory.StartNew(() => { Parallel.ForEach(archive.FileUnits, unit => { var currentFile = SrcMLElement.GetFileNameForUnit(unit); var language = SrcMLElement.GetLanguageForUnit(unit); try { var scopeForUnit = CodeParser[language].ParseFileUnit(unit); bc.Add(scopeForUnit); } catch(Exception e) { var key = e.StackTrace.Split('\n')[0].Trim(); if(!errors.ContainsKey(key)) { errors[key] = new List<string>(); } errors[key].Add(currentFile); } }); bc.CompleteAdding(); }); foreach(var item in bc.GetConsumingEnumerable()) { if(null == globalScope) { globalScope = item; } else { try { globalScope = globalScope.Merge(item); } catch(Exception e) { //So far, don't know how to log the error (ZL 04/2013) //Console.WriteLine("error"); } } } } } sw.Stop(); Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, sw.Elapsed, numberOfFailures); Console.WriteLine("\nSummary"); Console.WriteLine("==================="); Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles); Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles); Console.WriteLine("{0} to generate data", sw.Elapsed); Console.WriteLine(fileLogPath); PrintScopeReport(globalScope); PrintMethodCallReport(globalScope, callLogPath); PrintErrorReport(errors); monitor.Dispose(); //Assert.AreEqual(numberOfFailures, (from e in errors.Values select e.Count).Sum()); //Assert.AreEqual(0, numberOfFailures); }
private void TestDataGeneration(string sourcePath, string dataPath, bool useAsyncMethods=false) { string fileLogPath = Path.Combine(dataPath, "parse.log"); string callLogPath = Path.Combine(dataPath, "methodcalls.log"); bool regenerateSrcML = shouldRegenerateSrcML; if(!Directory.Exists(sourcePath)) { Assert.Ignore("Source code for is missing"); } if(File.Exists(callLogPath)) { File.Delete(callLogPath); } if(File.Exists(fileLogPath)) { File.Delete(fileLogPath); } if(!Directory.Exists(dataPath)) { regenerateSrcML = true; } else if(shouldRegenerateSrcML) { Directory.Delete(dataPath, true); } var archive = new SrcMLArchive(dataPath, regenerateSrcML); archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus; archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus; AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive); monitor.UseAsyncMethods = useAsyncMethods; ManualResetEvent mre = new ManualResetEvent(false); DateTime start, end = DateTime.MinValue; bool startupCompleted = false; //monitor.IsReadyChanged += (o, e) => { archive.IsReadyChanged += (o, e) => { if(e.ReadyState) { end = DateTime.Now; startupCompleted = true; mre.Set(); } }; start = DateTime.Now; monitor.Startup(); startupCompleted = mre.WaitOne(120000); if(!startupCompleted) { end = DateTime.Now; } Console.WriteLine("{0} to {1} srcML", end - start, (regenerateSrcML ? "generate" : "verify")); Assert.That(startupCompleted); Scope globalScope = null; int numberOfFailures = 0; int numberOfSuccesses = 0; int numberOfFiles = 0; Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>(); start = DateTime.Now; using(var fileLog = new StreamWriter(fileLogPath)) { foreach(var unit in archive.FileUnits) { if(++numberOfFiles % 100 == 0) { Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures); } var fileName = SrcMLElement.GetFileNameForUnit(unit); var language = SrcMLElement.GetLanguageForUnit(unit); fileLog.Write("Parsing {0}", fileName); try { var scopeForUnit = CodeParser[language].ParseFileUnit(unit); if(null == globalScope) { globalScope = scopeForUnit; } else { globalScope = globalScope.Merge(scopeForUnit); } fileLog.WriteLine(" PASSED"); numberOfSuccesses++; } catch(Exception e) { fileLog.WriteLine(" FAILED"); fileLog.WriteLine(e.StackTrace); var key = e.StackTrace.Split('\n')[0].Trim(); if(!errors.ContainsKey(key)) { errors[key] = new List<string>(); } errors[key].Add(fileName); numberOfFailures++; } } } end = DateTime.Now; Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, end - start, numberOfFailures); Console.WriteLine("\nSummary"); Console.WriteLine("==================="); Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles); Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles); Console.WriteLine("{0} to generate data", end - start); Console.WriteLine(fileLogPath); PrintScopeReport(globalScope); PrintMethodCallReport(globalScope, callLogPath); PrintErrorReport(errors); monitor.Dispose(); //Assert.AreEqual(numberOfFailures, (from e in errors.Values select e.Count).Sum()); //Assert.AreEqual(0, numberOfFailures); }