public MainWindow()
{
var archivePath = FSPath.Combine(".", "ARCHIVE");
_monitor = new FileSystemFolderMonitor(".", archivePath, new LastModifiedArchive(archivePath), new SrcMLArchive(archivePath));
_monitor.FileChanged += _archive_SourceFileChanged;
_monitor.StartMonitoring();
InitializeComponent();
}
private static void GenerateData(string sourcePath, string dataPath, string csvDirectory) {
Dictionary<Language, AbstractCodeParser> CodeParser = new Dictionary<Language, AbstractCodeParser>() {
{ Language.CPlusPlus, new CPlusPlusCodeParser() },
{ Language.Java, new JavaCodeParser() },
{ Language.CSharp, new CSharpCodeParser() }
};
string fileLogPath = Path.Combine(dataPath, "parse.log");
string callLogPath = Path.Combine(dataPath, "methodcalls.log");
string csvPath = Path.Combine(csvDirectory, "timing.csv");
string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23)));
if(!Directory.Exists(sourcePath)) {
Console.Error.WriteLine("{0} does not exist", sourcePath);
return;
}
if(File.Exists(callLogPath)) {
File.Delete(callLogPath);
}
if(File.Exists(fileLogPath)) {
File.Delete(fileLogPath);
}
var archive = new SrcMLArchive(dataPath);
archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus;
AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive);
ManualResetEvent mre = new ManualResetEvent(false);
Stopwatch timer = new Stopwatch();
bool startupCompleted = false;
monitor.IsReadyChanged += (o, e) => {
if(e.ReadyState) {
timer.Stop();
startupCompleted = true;
mre.Set();
}
};
timer.Start();
monitor.Startup();
string[] spinner = new string[] { "\\\r", "|\r", "/\r" };
int spinner_index = -1;
while(!startupCompleted) {
spinner_index = (++spinner_index) % 3;
Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]);
startupCompleted = mre.WaitOne(5000);
}
timer.Stop();
Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed);
Scope globalScope = null;
timer.Reset();
int numberOfFailures = 0;
int numberOfSuccesses = 0;
int numberOfFiles = 0;
Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>();
if(!File.Exists(csvPath)) {
File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") });
}
using(StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) {
timer.Start();
foreach(var unit in archive.FileUnits) {
var fileName = SrcMLElement.GetFileNameForUnit(unit);
var language = SrcMLElement.GetLanguageForUnit(unit);
try {
var scopeForUnit = CodeParser[language].ParseFileUnit(unit);
if(null == globalScope) {
globalScope = scopeForUnit;
} else {
globalScope = globalScope.Merge(scopeForUnit);
}
timer.Stop();
fileLog.WriteLine("Parsing {0} PASSED", fileName);
numberOfSuccesses++;
} catch(Exception e) {
timer.Stop();
fileLog.WriteLine("Parsing {0} FAILED", fileName);
fileLog.WriteLine(e.StackTrace);
var key = e.StackTrace.Split('\n')[0].Trim();
if(!errors.ContainsKey(key)) {
errors[key] = new List<string>();
}
errors[key].Add(fileName);
numberOfFailures++;
}
finally {
if(++numberOfFiles % 50 == 0) {
Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures);
csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds));
}
timer.Start();
}
}
}
timer.Stop();
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures);
Console.WriteLine("\nSummary");
Console.WriteLine("===================");
Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
Console.WriteLine("{0} to generate data", timer.Elapsed);
Console.WriteLine("See parse log at {0}", fileLogPath);
OutputCallGraphByType(globalScope, jsonPath);
PrintScopeReport(globalScope, sourcePath, csvDirectory);
PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath);
}
private void TestDataGeneration(string sourcePath, string dataPath, bool useAsyncMethods = false)
{
string fileLogPath = Path.Combine(dataPath, "parse.log");
string callLogPath = Path.Combine(dataPath, "methodcalls.log");
bool regenerateSrcML = shouldRegenerateSrcML;
if (!Directory.Exists(sourcePath))
{
Assert.Ignore("Source code for is missing");
}
if (File.Exists(callLogPath))
{
File.Delete(callLogPath);
}
if (File.Exists(fileLogPath))
{
File.Delete(fileLogPath);
}
if (!Directory.Exists(dataPath))
{
regenerateSrcML = true;
}
else if (shouldRegenerateSrcML)
{
Directory.Delete(dataPath, true);
}
var archive = new SrcMLArchive(dataPath, regenerateSrcML);
archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
int numberOfFailures = 0;
int numberOfSuccesses = 0;
int numberOfFiles = 0;
Dictionary <string, List <string> > errors = new Dictionary <string, List <string> >();
using (var fileLog = new StreamWriter(fileLogPath)) {
using (var monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive)) {
DateTime start, end = DateTime.MinValue;
bool startupCompleted = false;
start = DateTime.Now;
if (useAsyncMethods)
{
var task = monitor.UpdateArchivesAsync().ContinueWith((t) => {
end = DateTime.Now;
startupCompleted = true;
});
task.Wait();
}
else
{
monitor.UpdateArchives();
end = DateTime.Now;
startupCompleted = true;
}
if (!startupCompleted)
{
end = DateTime.Now;
}
Console.WriteLine("{0} to {1} srcML", end - start, (regenerateSrcML ? "generate" : "verify"));
Assert.That(startupCompleted);
using (var data = new DataRepository(archive)) {
start = DateTime.Now;
data.FileProcessed += (o, e) => {
if (e.EventType == FileEventType.FileAdded)
{
numberOfFiles++;
numberOfSuccesses++;
if (numberOfFiles % 100 == 0)
{
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures);
}
fileLog.WriteLine("OK {0}", e.FilePath);
}
};
data.ErrorRaised += (o, e) => {
numberOfFiles++;
numberOfFailures++;
if (numberOfFiles % 100 == 0)
{
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures);
}
ParseException pe = e.Exception as ParseException;
if (pe != null)
{
fileLog.WriteLine("ERROR {0}", pe.FileName);
fileLog.WriteLine(e.Exception.InnerException.StackTrace);
var key = e.Exception.InnerException.StackTrace.Split('\n')[0].Trim();
if (!errors.ContainsKey(key))
{
errors[key] = new List <string>();
}
int errorLineNumber = (pe.LineNumber < 1 ? 1 : pe.LineNumber);
int errorColumnNumber = (pe.ColumnNumber < 1 ? 1 : pe.ColumnNumber);
var errorLocation = String.Format("{0}({1},{2})", pe.FileName, errorLineNumber, errorColumnNumber);
errors[key].Add(errorLocation);
}
};
if (useAsyncMethods)
{
data.InitializeDataAsync().Wait();
}
else
{
data.InitializeData();
}
end = DateTime.Now;
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, end - start, numberOfFailures);
Console.WriteLine("{0} to generate data", end - start);
Console.WriteLine("\nSummary");
Console.WriteLine("===================");
Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
Console.WriteLine("{0} to generate data", end - start);
Console.WriteLine(fileLogPath);
IScope globalScope;
Assert.That(data.TryLockGlobalScope(Timeout.Infinite, out globalScope));
try {
PrintScopeReport(globalScope);
PrintMethodCallReport(globalScope, callLogPath);
PrintErrorReport(errors);
} finally {
data.ReleaseGlobalScopeLock();
}
}
}
}
}
private static void GenerateData(string sourcePath, string dataPath, string csvDirectory)
{
Dictionary <Language, ICodeParser> CodeParser = new Dictionary <Language, ICodeParser>()
{
{ Language.CPlusPlus, new CPlusPlusCodeParser() },
{ Language.Java, new JavaCodeParser() },
{ Language.CSharp, new CSharpCodeParser() }
};
string fileLogPath = Path.Combine(dataPath, "parse.log");
string callLogPath = Path.Combine(dataPath, "methodcalls.log");
string csvPath = Path.Combine(csvDirectory, "timing.csv");
string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23)));
if (!Directory.Exists(sourcePath))
{
Console.Error.WriteLine("{0} does not exist", sourcePath);
return;
}
if (File.Exists(callLogPath))
{
File.Delete(callLogPath);
}
if (File.Exists(fileLogPath))
{
File.Delete(fileLogPath);
}
var archive = new SrcMLArchive(dataPath);
archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus;
AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive);
ManualResetEvent mre = new ManualResetEvent(false);
Stopwatch timer = new Stopwatch();
bool startupCompleted = false;
monitor.UpdateArchivesCompleted += (o, e) => {
timer.Stop();
startupCompleted = true;
mre.Set();
};
timer.Start();
var task = monitor.UpdateArchivesAsync();
string[] spinner = new string[] { "\\\r", "|\r", "/\r" };
int spinner_index = -1;
while (!startupCompleted)
{
spinner_index = (++spinner_index) % 3;
Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]);
startupCompleted = mre.WaitOne(5000);
}
timer.Stop();
Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed);
IScope globalScope = null;
timer.Reset();
int numberOfFailures = 0;
int numberOfSuccesses = 0;
int numberOfFiles = 0;
Dictionary <string, List <string> > errors = new Dictionary <string, List <string> >();
if (!File.Exists(csvPath))
{
File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") });
}
using (StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) {
timer.Start();
foreach (var unit in archive.FileUnits)
{
var fileName = SrcMLElement.GetFileNameForUnit(unit);
var language = SrcMLElement.GetLanguageForUnit(unit);
try {
var scopeForUnit = CodeParser[language].ParseFileUnit(unit);
if (null == globalScope)
{
globalScope = scopeForUnit;
}
else
{
globalScope = globalScope.Merge(scopeForUnit);
}
timer.Stop();
fileLog.WriteLine("Parsing {0} PASSED", fileName);
numberOfSuccesses++;
} catch (Exception e) {
timer.Stop();
fileLog.WriteLine("Parsing {0} FAILED", fileName);
fileLog.WriteLine(e.StackTrace);
var key = e.StackTrace.Split('\n')[0].Trim();
if (!errors.ContainsKey(key))
{
errors[key] = new List <string>();
}
errors[key].Add(fileName);
numberOfFailures++;
} finally {
if (++numberOfFiles % 50 == 0)
{
Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures);
csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds));
}
timer.Start();
}
}
}
timer.Stop();
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures);
Console.WriteLine("\nSummary");
Console.WriteLine("===================");
Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
Console.WriteLine("{0} to generate data", timer.Elapsed);
Console.WriteLine("See parse log at {0}", fileLogPath);
OutputCallGraphByType(globalScope, jsonPath);
PrintScopeReport(globalScope, sourcePath, csvDirectory);
PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath);
}
private void TestDataGeneration_Concurrent(string sourcePath, string dataPath) {
string fileLogPath = Path.Combine(dataPath, "parse.log");
string callLogPath = Path.Combine(dataPath, "methodcalls.log");
bool regenerateSrcML = shouldRegenerateSrcML;
if(!Directory.Exists(sourcePath)) {
Assert.Ignore("Source code for is missing");
}
if(File.Exists(callLogPath)) {
File.Delete(callLogPath);
}
if(File.Exists(fileLogPath)) {
File.Delete(fileLogPath);
}
if(!Directory.Exists(dataPath)) {
regenerateSrcML = true;
} else if(shouldRegenerateSrcML) {
Directory.Delete(dataPath, true);
}
var archive = new SrcMLArchive(dataPath, regenerateSrcML);
archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive);
ManualResetEvent mre = new ManualResetEvent(false);
Stopwatch sw = new Stopwatch();
bool startupCompleted = false;
monitor.IsReadyChanged += (o, e) => {
if(e.ReadyState) {
sw.Stop();
startupCompleted = true;
mre.Set();
}
};
sw.Start();
//add the concurrency test: ZL
monitor.Startup_Concurrent();
startupCompleted = mre.WaitOne(60000);
if(sw.IsRunning) {
sw.Stop();
}
Console.WriteLine("{0} to {1} srcML", sw.Elapsed, (regenerateSrcML ? "generate" : "verify"));
Assert.That(startupCompleted);
Scope globalScope = null;
sw.Reset();
int numberOfFailures = 0;
int numberOfSuccesses = 0;
int numberOfFiles = 0;
Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>();
sw.Start();
using(var fileLog = new StreamWriter(fileLogPath)) {
using(BlockingCollection<NamespaceDefinition> bc = new BlockingCollection<NamespaceDefinition>()) {
Task.Factory.StartNew(() => {
Parallel.ForEach(archive.FileUnits, unit => {
var currentFile = SrcMLElement.GetFileNameForUnit(unit);
var language = SrcMLElement.GetLanguageForUnit(unit);
try {
var scopeForUnit = CodeParser[language].ParseFileUnit(unit);
bc.Add(scopeForUnit);
} catch(Exception e) {
var key = e.StackTrace.Split('\n')[0].Trim();
if(!errors.ContainsKey(key)) {
errors[key] = new List<string>();
}
errors[key].Add(currentFile);
}
});
bc.CompleteAdding();
});
foreach(var item in bc.GetConsumingEnumerable()) {
if(null == globalScope) {
globalScope = item;
} else {
try {
globalScope = globalScope.Merge(item);
} catch(Exception e) {
//So far, don't know how to log the error (ZL 04/2013)
//Console.WriteLine("error");
}
}
}
}
}
sw.Stop();
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, sw.Elapsed, numberOfFailures);
Console.WriteLine("\nSummary");
Console.WriteLine("===================");
Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
Console.WriteLine("{0} to generate data", sw.Elapsed);
Console.WriteLine(fileLogPath);
PrintScopeReport(globalScope);
PrintMethodCallReport(globalScope, callLogPath);
PrintErrorReport(errors);
monitor.Dispose();
//Assert.AreEqual(numberOfFailures, (from e in errors.Values select e.Count).Sum());
//Assert.AreEqual(0, numberOfFailures);
}
private void TestDataGeneration(string sourcePath, string dataPath, bool useAsyncMethods=false) {
string fileLogPath = Path.Combine(dataPath, "parse.log");
string callLogPath = Path.Combine(dataPath, "methodcalls.log");
bool regenerateSrcML = shouldRegenerateSrcML;
if(!Directory.Exists(sourcePath)) {
Assert.Ignore("Source code for is missing");
}
if(File.Exists(callLogPath)) {
File.Delete(callLogPath);
}
if(File.Exists(fileLogPath)) {
File.Delete(fileLogPath);
}
if(!Directory.Exists(dataPath)) {
regenerateSrcML = true;
} else if(shouldRegenerateSrcML) {
Directory.Delete(dataPath, true);
}
var archive = new SrcMLArchive(dataPath, regenerateSrcML);
archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive);
monitor.UseAsyncMethods = useAsyncMethods;
ManualResetEvent mre = new ManualResetEvent(false);
DateTime start, end = DateTime.MinValue;
bool startupCompleted = false;
//monitor.IsReadyChanged += (o, e) => {
archive.IsReadyChanged += (o, e) => {
if(e.ReadyState) {
end = DateTime.Now;
startupCompleted = true;
mre.Set();
}
};
start = DateTime.Now;
monitor.Startup();
startupCompleted = mre.WaitOne(120000);
if(!startupCompleted) {
end = DateTime.Now;
}
Console.WriteLine("{0} to {1} srcML", end - start, (regenerateSrcML ? "generate" : "verify"));
Assert.That(startupCompleted);
Scope globalScope = null;
int numberOfFailures = 0;
int numberOfSuccesses = 0;
int numberOfFiles = 0;
Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>();
start = DateTime.Now;
using(var fileLog = new StreamWriter(fileLogPath)) {
foreach(var unit in archive.FileUnits) {
if(++numberOfFiles % 100 == 0) {
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures);
}
var fileName = SrcMLElement.GetFileNameForUnit(unit);
var language = SrcMLElement.GetLanguageForUnit(unit);
fileLog.Write("Parsing {0}", fileName);
try {
var scopeForUnit = CodeParser[language].ParseFileUnit(unit);
if(null == globalScope) {
globalScope = scopeForUnit;
} else {
globalScope = globalScope.Merge(scopeForUnit);
}
fileLog.WriteLine(" PASSED");
numberOfSuccesses++;
} catch(Exception e) {
fileLog.WriteLine(" FAILED");
fileLog.WriteLine(e.StackTrace);
var key = e.StackTrace.Split('\n')[0].Trim();
if(!errors.ContainsKey(key)) {
errors[key] = new List<string>();
}
errors[key].Add(fileName);
numberOfFailures++;
}
}
}
end = DateTime.Now;
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, end - start, numberOfFailures);
Console.WriteLine("\nSummary");
Console.WriteLine("===================");
Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
Console.WriteLine("{0} to generate data", end - start);
Console.WriteLine(fileLogPath);
PrintScopeReport(globalScope);
PrintMethodCallReport(globalScope, callLogPath);
PrintErrorReport(errors);
monitor.Dispose();
//Assert.AreEqual(numberOfFailures, (from e in errors.Values select e.Count).Sum());
//Assert.AreEqual(0, numberOfFailures);
}
