private void TestDataGeneration(string sourcePath, string dataPath, bool useAsyncMethods = false)
{
string fileLogPath = Path.Combine(dataPath, "parse.log");
string callLogPath = Path.Combine(dataPath, "methodcalls.log");
bool regenerateSrcML = shouldRegenerateSrcML;
if (!Directory.Exists(sourcePath))
{
Assert.Ignore("Source code for is missing");
}
if (File.Exists(callLogPath))
{
File.Delete(callLogPath);
}
if (File.Exists(fileLogPath))
{
File.Delete(fileLogPath);
}
if (!Directory.Exists(dataPath))
{
regenerateSrcML = true;
}
else if (shouldRegenerateSrcML)
{
Directory.Delete(dataPath, true);
}
var archive = new SrcMLArchive(dataPath, regenerateSrcML);
archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
int numberOfFailures = 0;
int numberOfSuccesses = 0;
int numberOfFiles = 0;
Dictionary <string, List <string> > errors = new Dictionary <string, List <string> >();
using (var fileLog = new StreamWriter(fileLogPath)) {
using (var monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive)) {
DateTime start, end = DateTime.MinValue;
bool startupCompleted = false;
start = DateTime.Now;
if (useAsyncMethods)
{
var task = monitor.UpdateArchivesAsync().ContinueWith((t) => {
end = DateTime.Now;
startupCompleted = true;
});
task.Wait();
}
else
{
monitor.UpdateArchives();
end = DateTime.Now;
startupCompleted = true;
}
if (!startupCompleted)
{
end = DateTime.Now;
}
Console.WriteLine("{0} to {1} srcML", end - start, (regenerateSrcML ? "generate" : "verify"));
Assert.That(startupCompleted);
using (var data = new DataRepository(archive)) {
start = DateTime.Now;
data.FileProcessed += (o, e) => {
if (e.EventType == FileEventType.FileAdded)
{
numberOfFiles++;
numberOfSuccesses++;
if (numberOfFiles % 100 == 0)
{
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures);
}
fileLog.WriteLine("OK {0}", e.FilePath);
}
};
data.ErrorRaised += (o, e) => {
numberOfFiles++;
numberOfFailures++;
if (numberOfFiles % 100 == 0)
{
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, DateTime.Now - start, numberOfFailures);
}
ParseException pe = e.Exception as ParseException;
if (pe != null)
{
fileLog.WriteLine("ERROR {0}", pe.FileName);
fileLog.WriteLine(e.Exception.InnerException.StackTrace);
var key = e.Exception.InnerException.StackTrace.Split('\n')[0].Trim();
if (!errors.ContainsKey(key))
{
errors[key] = new List <string>();
}
int errorLineNumber = (pe.LineNumber < 1 ? 1 : pe.LineNumber);
int errorColumnNumber = (pe.ColumnNumber < 1 ? 1 : pe.ColumnNumber);
var errorLocation = String.Format("{0}({1},{2})", pe.FileName, errorLineNumber, errorColumnNumber);
errors[key].Add(errorLocation);
}
};
if (useAsyncMethods)
{
data.InitializeDataAsync().Wait();
}
else
{
data.InitializeData();
}
end = DateTime.Now;
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, end - start, numberOfFailures);
Console.WriteLine("{0} to generate data", end - start);
Console.WriteLine("\nSummary");
Console.WriteLine("===================");
Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
Console.WriteLine("{0} to generate data", end - start);
Console.WriteLine(fileLogPath);
IScope globalScope;
Assert.That(data.TryLockGlobalScope(Timeout.Infinite, out globalScope));
try {
PrintScopeReport(globalScope);
PrintMethodCallReport(globalScope, callLogPath);
PrintErrorReport(errors);
} finally {
data.ReleaseGlobalScopeLock();
}
}
}
}
}
private static void GenerateData(string sourcePath, string dataPath, string csvDirectory)
{
Dictionary <Language, ICodeParser> CodeParser = new Dictionary <Language, ICodeParser>()
{
{ Language.CPlusPlus, new CPlusPlusCodeParser() },
{ Language.Java, new JavaCodeParser() },
{ Language.CSharp, new CSharpCodeParser() }
};
string fileLogPath = Path.Combine(dataPath, "parse.log");
string callLogPath = Path.Combine(dataPath, "methodcalls.log");
string csvPath = Path.Combine(csvDirectory, "timing.csv");
string jsonPath = String.Format("{0}.json", Path.Combine(@"c:\Workspace\DataVisualization", dataPath.Substring(23)));
if (!Directory.Exists(sourcePath))
{
Console.Error.WriteLine("{0} does not exist", sourcePath);
return;
}
if (File.Exists(callLogPath))
{
File.Delete(callLogPath);
}
if (File.Exists(fileLogPath))
{
File.Delete(fileLogPath);
}
var archive = new SrcMLArchive(dataPath);
archive.XmlGenerator.ExtensionMapping[".cxx"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".c"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".cc"] = Language.CPlusPlus;
archive.XmlGenerator.ExtensionMapping[".hpp"] = Language.CPlusPlus;
AbstractFileMonitor monitor = new FileSystemFolderMonitor(sourcePath, dataPath, new LastModifiedArchive(dataPath), archive);
ManualResetEvent mre = new ManualResetEvent(false);
Stopwatch timer = new Stopwatch();
bool startupCompleted = false;
monitor.UpdateArchivesCompleted += (o, e) => {
timer.Stop();
startupCompleted = true;
mre.Set();
};
timer.Start();
var task = monitor.UpdateArchivesAsync();
string[] spinner = new string[] { "\\\r", "|\r", "/\r" };
int spinner_index = -1;
while (!startupCompleted)
{
spinner_index = (++spinner_index) % 3;
Console.Write("Updating archive for {0}... {1}", sourcePath, spinner[spinner_index]);
startupCompleted = mre.WaitOne(5000);
}
timer.Stop();
Console.WriteLine("Updating archive for {0}... {1}", sourcePath, timer.Elapsed);
IScope globalScope = null;
timer.Reset();
int numberOfFailures = 0;
int numberOfSuccesses = 0;
int numberOfFiles = 0;
Dictionary <string, List <string> > errors = new Dictionary <string, List <string> >();
if (!File.Exists(csvPath))
{
File.WriteAllLines(csvPath, new string[] { String.Join(",", "Project", "Files", "Failures", "Time (s)") });
}
using (StreamWriter fileLog = new StreamWriter(fileLogPath), csvFile = new StreamWriter(csvPath, true)) {
timer.Start();
foreach (var unit in archive.FileUnits)
{
var fileName = SrcMLElement.GetFileNameForUnit(unit);
var language = SrcMLElement.GetLanguageForUnit(unit);
try {
var scopeForUnit = CodeParser[language].ParseFileUnit(unit);
if (null == globalScope)
{
globalScope = scopeForUnit;
}
else
{
globalScope = globalScope.Merge(scopeForUnit);
}
timer.Stop();
fileLog.WriteLine("Parsing {0} PASSED", fileName);
numberOfSuccesses++;
} catch (Exception e) {
timer.Stop();
fileLog.WriteLine("Parsing {0} FAILED", fileName);
fileLog.WriteLine(e.StackTrace);
var key = e.StackTrace.Split('\n')[0].Trim();
if (!errors.ContainsKey(key))
{
errors[key] = new List <string>();
}
errors[key].Add(fileName);
numberOfFailures++;
} finally {
if (++numberOfFiles % 50 == 0)
{
Console.Write("{0,5:N0} files completed in {1} with {2,5:N0} failures\r", numberOfFiles, timer.Elapsed, numberOfFailures);
csvFile.WriteLine(string.Join(",", sourcePath, numberOfFiles, numberOfFailures, timer.Elapsed.TotalSeconds));
}
timer.Start();
}
}
}
timer.Stop();
Console.WriteLine("{0,5:N0} files completed in {1} with {2,5:N0} failures", numberOfFiles, timer.Elapsed, numberOfFailures);
Console.WriteLine("\nSummary");
Console.WriteLine("===================");
Console.WriteLine("{0,10:N0} failures ({1,8:P2})", numberOfFailures, ((float)numberOfFailures) / numberOfFiles);
Console.WriteLine("{0,10:N0} successes ({1,8:P2})", numberOfSuccesses, ((float)numberOfSuccesses) / numberOfFiles);
Console.WriteLine("{0} to generate data", timer.Elapsed);
Console.WriteLine("See parse log at {0}", fileLogPath);
OutputCallGraphByType(globalScope, jsonPath);
PrintScopeReport(globalScope, sourcePath, csvDirectory);
PrintMethodCallReport(globalScope, sourcePath, csvDirectory, callLogPath);
}
