private static void GenerateData(string sourcePath, string dataPath, string csvDirectory) {
Dictionary<Language, AbstractCodeParser> CodeParser = new Dictionary<Language, AbstractCodeParser>() {
{ Language.CPlusPlus, new CPlusPlusCodeParser() },
{ Language.Java, new JavaCodeParser() },
{ Language.CSharp, new CSharpCodeParser() }
};
string fileLogPath = Path.Combine(dataPath, "parse.log");
string callLogPath = Path.Combine(dataPath, "methodcalls.log");
string csvPath = Path.Combine(csvDirectory, "timing.csv");
string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23)));
if(!Directory.Exists(sourcePath)) {
Console.Error.WriteLine("{0} does not exist", sourcePath);
return;
}
if(File.Exists(callLogPath)) {
File.Delete(callLogPath);
}
if(File.Exists(fileLogPath)) {
File.Delete(fileLogPath);
}
var archive = new SrcMLArchive(dataPath);
archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus;
AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive);
ManualResetEvent mre = new ManualResetEvent(false);
Stopwatch timer = new Stopwatch();
bool startupCompleted = false;
monitor.IsReadyChanged += (o, e) => {
if(e.ReadyState) {
timer.Stop();
startupCompleted = true;
mre.Set();
}
};
timer.Start();
monitor.Startup();
string[] spinner = new string[] { "\\\r", "|\r", "/\r" };
int spinner_index = -1;
while(!startupCompleted) {
spinner_index = (++spinner_index) % 3;
Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]);
startupCompleted = mre.WaitOne(5000);
}
timer.Stop();
Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed);
Scope globalScope = null;
timer.Reset();
int numberOfFailures = 0;
int numberOfSuccesses = 0;
int numberOfFiles = 0;
Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>();
if(!File.Exists(csvPath)) {
File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") });
}
using(StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) {
timer.Start();
foreach(var unit in archive.FileUnits) {
var fileName = SrcMLElement.GetFileNameForUnit(unit);
var language = SrcMLElement.GetLanguageForUnit(unit);
try {
var scopeForUnit = CodeParser[language].ParseFileUnit(unit);
if(null == globalScope) {
globalScope = scopeForUnit;
} else {
globalScope = globalScope.Merge(scopeForUnit);
}
timer.Stop();
fileLog.WriteLine("Parsing {0} PASSED", fileName);
numberOfSuccesses++;
} catch(Exception e) {
timer.Stop();
fileLog.WriteLine("Parsing {0} FAILED", fileName);
fileLog.WriteLine(e.StackTrace);
var key = e.StackTrace.Split('\n')[0].Trim();
if(!errors.ContainsKey(key)) {
errors[key] = new List<string>();
}
errors[key].Add(fileName);
numberOfFailures++;
}
finally {
if(++numberOfFiles % 50 == 0) {
Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures);
csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds));
}
timer.Start();
}
}
}
timer.Stop();
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures);
Console.WriteLine("\nSummary");
Console.WriteLine("===================");
Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
Console.WriteLine("{0} to generate data", timer.Elapsed);
Console.WriteLine("See parse log at {0}", fileLogPath);
OutputCallGraphByType(globalScope, jsonPath);
PrintScopeReport(globalScope, sourcePath, csvDirectory);
PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath);
}
private void TestDataGeneration(string sourcePath, string dataPath, bool useAsyncMethods=false) {
string fileLogPath = Path.Combine(dataPath, "parse.log");
string callLogPath = Path.Combine(dataPath, "methodcalls.log");
bool regenerateSrcML = shouldRegenerateSrcML;
if(!Directory.Exists(sourcePath)) {
Assert.Ignore("Source code for is missing");
}
if(File.Exists(callLogPath)) {
File.Delete(callLogPath);
}
if(File.Exists(fileLogPath)) {
File.Delete(fileLogPath);
}
if(!Directory.Exists(dataPath)) {
regenerateSrcML = true;
} else if(shouldRegenerateSrcML) {
Directory.Delete(dataPath, true);
}
var archive = new SrcMLArchive(dataPath, regenerateSrcML);
archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive);
monitor.UseAsyncMethods = useAsyncMethods;
ManualResetEvent mre = new ManualResetEvent(false);
DateTime start, end = DateTime.MinValue;
bool startupCompleted = false;
//monitor.IsReadyChanged += (o, e) => {
archive.IsReadyChanged += (o, e) => {
if(e.ReadyState) {
end = DateTime.Now;
startupCompleted = true;
mre.Set();
}
};
start = DateTime.Now;
monitor.Startup();
startupCompleted = mre.WaitOne(120000);
if(!startupCompleted) {
end = DateTime.Now;
}
Console.WriteLine("{0} to {1} srcML", end - start, (regenerateSrcML ? "generate" : "verify"));
Assert.That(startupCompleted);
Scope globalScope = null;
int numberOfFailures = 0;
int numberOfSuccesses = 0;
int numberOfFiles = 0;
Dictionary<string, List<string>> errors = new Dictionary<string, List<string>>();
start = DateTime.Now;
using(var fileLog = new StreamWriter(fileLogPath)) {
foreach(var unit in archive.FileUnits) {
if(++numberOfFiles % 100 == 0) {
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures);
}
var fileName = SrcMLElement.GetFileNameForUnit(unit);
var language = SrcMLElement.GetLanguageForUnit(unit);
fileLog.Write("Parsing {0}", fileName);
try {
var scopeForUnit = CodeParser[language].ParseFileUnit(unit);
if(null == globalScope) {
globalScope = scopeForUnit;
} else {
globalScope = globalScope.Merge(scopeForUnit);
}
fileLog.WriteLine(" PASSED");
numberOfSuccesses++;
} catch(Exception e) {
fileLog.WriteLine(" FAILED");
fileLog.WriteLine(e.StackTrace);
var key = e.StackTrace.Split('\n')[0].Trim();
if(!errors.ContainsKey(key)) {
errors[key] = new List<string>();
}
errors[key].Add(fileName);
numberOfFailures++;
}
}
}
end = DateTime.Now;
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, end - start, numberOfFailures);
Console.WriteLine("\nSummary");
Console.WriteLine("===================");
Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
Console.WriteLine("{0} to generate data", end - start);
Console.WriteLine(fileLogPath);
PrintScopeReport(globalScope);
PrintMethodCallReport(globalScope, callLogPath);
PrintErrorReport(errors);
monitor.Dispose();
//Assert.AreEqual(numberOfFailures, (from e in errors.Values select e.Count).Sum());
//Assert.AreEqual(0, numberOfFailures);
}