public bool Run(TestInfo test, int pass, int testNumber)
{
if (_showStatus)
{
Log("#@ Running {0} ({1})...\n", test.TestMethod.Name, Language.TwoLetterISOLanguageName);
}
if (_buildMode)
{
Log("{0,3}. {1,-46} ",
testNumber,
test.TestMethod.Name);
}
else
{
var time = DateTime.Now;
Log("[{0}:{1}] {2,3}.{3,-3} {4,-46} ({5}) ",
time.Hour.ToString("D2"),
time.Minute.ToString("D2"),
pass,
testNumber,
test.TestMethod.Name,
Language.TwoLetterISOLanguageName);
}
// Create test class.
var testObject = Activator.CreateInstance(test.TestClassType);
// Set the TestContext.
TestContext.Properties["AccessInternet"] = AccessInternet.ToString();
TestContext.Properties["RunPerfTests"] = RunPerfTests.ToString();
TestContext.Properties["TestSmallMolecules"] = TestSmallMolecules.ToString();
TestContext.Properties["LiveReports"] = LiveReports.ToString();
if (test.SetTestContext != null)
{
var context = new object[] { TestContext };
test.SetTestContext.Invoke(testObject, context);
}
// Switch to selected culture.
var saveCulture = Thread.CurrentThread.CurrentCulture;
var saveUICulture = Thread.CurrentThread.CurrentUICulture;
LocalizationHelper.CurrentCulture = LocalizationHelper.CurrentUICulture = Language;
LocalizationHelper.InitThread();
long crtLeakedBytes = 0;
// Run the test and time it.
Exception exception = null;
var stopwatch = new Stopwatch();
stopwatch.Start();
try
{
if (test.TestInitialize != null)
{
test.TestInitialize.Invoke(testObject, null);
}
if (CheckCrtLeaks > 0)
{
CrtDebugHeap.Checkpoint();
}
test.TestMethod.Invoke(testObject, null);
if (CheckCrtLeaks > 0)
{
crtLeakedBytes = CrtDebugHeap.DumpLeaks(true);
}
if (test.TestCleanup != null)
{
test.TestCleanup.Invoke(testObject, null);
}
}
catch (Exception e)
{
exception = e;
}
stopwatch.Stop();
LastTestDuration = (int)(stopwatch.ElapsedMilliseconds / 1000);
// Restore culture.
Thread.CurrentThread.CurrentCulture = saveCulture;
Thread.CurrentThread.CurrentUICulture = saveUICulture;
MemoryManagement.FlushMemory();
const int mb = 1024 * 1024;
var managedMemory = (double)GC.GetTotalMemory(true) / mb;
if (exception == null)
{
// Test succeeded.
Log(
"{0,3} failures, {1:F2}/{2:F1} MB, {3} sec.\r\n",
FailureCount,
managedMemory,
TotalMemory,
LastTestDuration);
if (crtLeakedBytes > CheckCrtLeaks)
{
Log("!!! {0} CRT-LEAKED {1} bytes", test.TestMethod.Name, crtLeakedBytes);
}
using (var writer = new FileStream("TestRunnerMemory.log", FileMode.Append, FileAccess.Write, FileShare.Read))
using (var stringWriter = new StreamWriter(writer))
{
stringWriter.WriteLine(TotalMemory.ToString("F1"));
}
return(true);
}
// Save failure information.
FailureCount++;
if (FailureCounts.ContainsKey(test.TestMethod.Name))
{
FailureCounts[test.TestMethod.Name]++;
}
else
{
FailureCounts[test.TestMethod.Name] = 1;
}
var failureInfo = "# " + test.TestMethod.Name + "FAILED:\n" +
exception.InnerException.Message + "\n" +
exception.InnerException.StackTrace;
if (ErrorCounts.ContainsKey(failureInfo))
{
ErrorCounts[failureInfo]++;
}
else
{
ErrorCounts[failureInfo] = 1;
}
Log(
"{0,3} failures, {1:F2}/{2:F1} MB\r\n\r\n!!! {3} FAILED\r\n{4}\r\n{5}\r\n!!!\r\n\r\n",
FailureCount, managedMemory, TotalMemory, test.TestMethod.Name,
exception.InnerException.Message,
exception.InnerException.StackTrace);
return(false);
}
public bool Run(TestInfo test, int pass, int testNumber)
{
if (_showStatus)
{
Log("#@ Running {0} ({1})...\n", test.TestMethod.Name, Language.TwoLetterISOLanguageName);
}
if (_buildMode)
{
Log("{0,3}. {1,-46} ",
testNumber,
test.TestMethod.Name);
}
else
{
var time = DateTime.Now;
Log("[{0}:{1}] {2,3}.{3,-3} {4,-46} ({5}) ",
time.Hour.ToString("D2"),
time.Minute.ToString("D2"),
pass,
testNumber,
test.TestMethod.Name,
Language.TwoLetterISOLanguageName);
}
Exception exception = null;
var stopwatch = new Stopwatch();
stopwatch.Start();
var saveCulture = Thread.CurrentThread.CurrentCulture;
var saveUICulture = Thread.CurrentThread.CurrentUICulture;
long crtLeakedBytes = 0;
try
{
// Create test class.
var testObject = Activator.CreateInstance(test.TestClassType);
// Set the TestContext.
TestContext.Properties["AccessInternet"] = AccessInternet.ToString();
TestContext.Properties["RunPerfTests"] = RunPerfTests.ToString();
TestContext.Properties["TestSmallMolecules"] = AddSmallMoleculeNodes.ToString(); // Add the magic small molecule test node to every document?
TestContext.Properties["RunSmallMoleculeTestVersions"] = RunsSmallMoleculeVersions.ToString(); // Run the AsSmallMolecule version of tests when available?
TestContext.Properties["LiveReports"] = LiveReports.ToString();
TestContext.Properties["TestName"] = test.TestMethod.Name;
TestContext.Properties["TestRunResultsDirectory"] =
Path.Combine(TestContext.TestDir, test.TestClassType.Name);
if (test.SetTestContext != null)
{
var context = new object[] { TestContext };
test.SetTestContext.Invoke(testObject, context);
}
// Switch to selected culture.
LocalizationHelper.CurrentCulture = LocalizationHelper.CurrentUICulture = Language;
LocalizationHelper.InitThread();
// Run the test and time it.
if (test.TestInitialize != null)
{
test.TestInitialize.Invoke(testObject, null);
}
if (CheckCrtLeaks > 0)
{
// TODO: CrtDebugHeap class used to be provided by Crawdad.dll
// If we ever want to enable this funcationality again, we need to find another .dll
// to put this in.
//CrtDebugHeap.Checkpoint();
}
test.TestMethod.Invoke(testObject, null);
if (CheckCrtLeaks > 0)
{
//crtLeakedBytes = CrtDebugHeap.DumpLeaks(true);
}
if (test.TestCleanup != null)
{
test.TestCleanup.Invoke(testObject, null);
}
}
catch (Exception e)
{
exception = e;
}
stopwatch.Stop();
LastTestDuration = (int)(stopwatch.ElapsedMilliseconds / 1000);
// Restore culture.
Thread.CurrentThread.CurrentCulture = saveCulture;
Thread.CurrentThread.CurrentUICulture = saveUICulture;
MemoryManagement.FlushMemory();
_process.Refresh();
ManagedMemoryBytes = GC.GetTotalMemory(true);
TotalMemoryBytes = _process.PrivateMemorySize64;
LastTotalHandleCount = GetHandleCount(HandleType.total);
LastUserHandleCount = GetHandleCount(HandleType.user);
LastGdiHandleCount = GetHandleCount(HandleType.gdi);
if (exception == null)
{
// Test succeeded.
Log("{0,3} failures, {1:F2}/{2:F1} MB, {3}/{4} handles, {5} sec.\r\n",
FailureCount,
ManagedMemory,
TotalMemory,
LastUserHandleCount + LastGdiHandleCount,
LastTotalHandleCount,
LastTestDuration);
if (crtLeakedBytes > CheckCrtLeaks)
{
Log("!!! {0} CRT-LEAKED {1} bytes\r\n", test.TestMethod.Name, crtLeakedBytes);
}
using (var writer = new FileStream("TestRunnerMemory.log", FileMode.Append, FileAccess.Write, FileShare.Read))
using (var stringWriter = new StreamWriter(writer))
{
stringWriter.WriteLine(TotalMemory.ToString("F1"));
}
return(true);
}
// Save failure information.
FailureCount++;
if (FailureCounts.ContainsKey(test.TestMethod.Name))
{
FailureCounts[test.TestMethod.Name]++;
}
else
{
FailureCounts[test.TestMethod.Name] = 1;
}
var message = exception.InnerException == null ? exception.Message : exception.InnerException.Message;
var stackTrace = exception.InnerException == null ? exception.StackTrace : exception.InnerException.StackTrace;
var failureInfo = "# " + test.TestMethod.Name + "FAILED:\n" +
message + "\n" +
stackTrace;
if (ErrorCounts.ContainsKey(failureInfo))
{
ErrorCounts[failureInfo]++;
}
else
{
ErrorCounts[failureInfo] = 1;
}
Log("{0,3} failures, {1:F2}/{2:F1} MB, {3}/{4} handles, {5} sec.\r\n\r\n!!! {6} FAILED\r\n{7}\r\n{8}\r\n!!!\r\n\r\n",
FailureCount,
ManagedMemory,
TotalMemory,
LastUserHandleCount + LastGdiHandleCount,
LastTotalHandleCount,
LastTestDuration,
test.TestMethod.Name,
message,
exception);
return(false);
}
public bool Run(TestInfo test, int pass, int testNumber, string dmpDir)
{
TeamCityStartTest(test);
if (_showStatus)
{
Log("#@ Running {0} ({1})...\n", test.TestMethod.Name, Language.TwoLetterISOLanguageName);
}
if (_buildMode)
{
Log("{0,3}. {1,-46} ",
testNumber,
test.TestMethod.Name);
}
else
{
var time = DateTime.Now;
Log("[{0}:{1}] {2,3}.{3,-3} {4,-46} ({5}) ",
time.Hour.ToString("D2"),
time.Minute.ToString("D2"),
pass,
testNumber,
test.TestMethod.Name,
Language.TwoLetterISOLanguageName);
}
Exception exception = null;
var stopwatch = new Stopwatch();
stopwatch.Start();
var saveCulture = Thread.CurrentThread.CurrentCulture;
var saveUICulture = Thread.CurrentThread.CurrentUICulture;
long crtLeakedBytes = 0;
var testResultsDir = Path.Combine(TestContext.TestDir, test.TestClassType.Name);
var dumpFileName = string.Format("{0}.{1}_{2}_{3}_{4:yyyy_MM_dd__hh_mm_ss_tt}.dmp", pass, testNumber, test.TestMethod.Name, Language.TwoLetterISOLanguageName, DateTime.Now);
if (test.MinidumpLeakThreshold != null)
{
try
{
if (string.IsNullOrEmpty(dmpDir))
{
dmpDir = Path.Combine(testResultsDir, "Minidumps");
Log("[WARNING] No log path provided - using test results dir ({0})", dmpDir);
}
Directory.CreateDirectory(dmpDir);
if (!MiniDump.WriteMiniDump(Path.Combine(dmpDir, "pre_" + dumpFileName)))
{
Log("[WARNING] Failed to write pre mini dump (GetLastError() = {0})", Marshal.GetLastWin32Error());
}
}
catch (Exception ex)
{
Log("[WARNING] Exception thrown when creating memory dump: {0}\r\n{1}\r\n", ex.InnerException?.Message ?? ex.Message, ex.InnerException?.StackTrace ?? ex.StackTrace);
}
}
try
{
// Create test class.
var testObject = Activator.CreateInstance(test.TestClassType);
// Set the TestContext.
TestContext.Properties["AccessInternet"] = AccessInternet.ToString();
TestContext.Properties["RunPerfTests"] = RunPerfTests.ToString();
TestContext.Properties["TestSmallMolecules"] = AddSmallMoleculeNodes.ToString(); // Add the magic small molecule test node to every document?
TestContext.Properties["RunSmallMoleculeTestVersions"] = RunsSmallMoleculeVersions.ToString(); // Run the AsSmallMolecule version of tests when available?
TestContext.Properties["LiveReports"] = LiveReports.ToString();
TestContext.Properties["TestName"] = test.TestMethod.Name;
TestContext.Properties["TestRunResultsDirectory"] = testResultsDir;
if (test.SetTestContext != null)
{
var context = new object[] { TestContext };
test.SetTestContext.Invoke(testObject, context);
}
// Switch to selected culture.
LocalizationHelper.CurrentCulture = LocalizationHelper.CurrentUICulture = Language;
LocalizationHelper.InitThread();
// Run the test and time it.
if (test.TestInitialize != null)
{
test.TestInitialize.Invoke(testObject, null);
}
if (CheckCrtLeaks > 0)
{
// TODO: CrtDebugHeap class used to be provided by Crawdad.dll
// If we ever want to enable this funcationality again, we need to find another .dll
// to put this in.
//CrtDebugHeap.Checkpoint();
}
test.TestMethod.Invoke(testObject, null);
if (CheckCrtLeaks > 0)
{
//crtLeakedBytes = CrtDebugHeap.DumpLeaks(true);
}
if (test.TestCleanup != null)
{
test.TestCleanup.Invoke(testObject, null);
}
}
catch (Exception e)
{
exception = e;
}
stopwatch.Stop();
LastTestDuration = (int)(stopwatch.ElapsedMilliseconds / 1000);
// Allow as much to be garbage collected as possible
// Restore culture.
Thread.CurrentThread.CurrentCulture = saveCulture;
Thread.CurrentThread.CurrentUICulture = saveUICulture;
MemoryManagement.FlushMemory();
_process.Refresh();
var heapCounts = ReportSystemHeaps ? MemoryManagement.GetProcessHeapSizes() : new MemoryManagement.HeapAllocationSizes[1];
var processBytes = heapCounts[0].Committed; // Process heap : useful for debugging - though included in committed bytes
var managedBytes = GC.GetTotalMemory(true); // Managed heap
var committedBytes = heapCounts.Sum(h => h.Committed);
ManagedMemoryBytes = managedBytes;
CommittedMemoryBytes = committedBytes;
var previousPrivateBytes = TotalMemoryBytes;
TotalMemoryBytes = _process.PrivateMemorySize64;
LastTotalHandleCount = GetHandleCount(HandleType.total);
LastUserHandleCount = GetHandleCount(HandleType.user);
LastGdiHandleCount = GetHandleCount(HandleType.gdi);
if (test.MinidumpLeakThreshold != null)
{
try
{
var leak = (TotalMemoryBytes - previousPrivateBytes) / MB;
if (leak > test.MinidumpLeakThreshold.Value)
{
if (!MiniDump.WriteMiniDump(Path.Combine(dmpDir, "post_" + dumpFileName)))
{
Log("[WARNING] Failed to write post mini dump (GetLastError() = {0})", Marshal.GetLastWin32Error());
}
}
else
{
var prePath = Path.Combine(dmpDir, "pre_" + dumpFileName);
var i = 5;
while (i-- > 0)
{
File.Delete(prePath);
if (!File.Exists(prePath))
{
break;
}
Thread.Sleep(200);
}
}
}
catch (Exception ex)
{
Log("[WARNING] Exception thrown when creating memory dump: {0}\r\n{1}\r\n", ex.InnerException?.Message ?? ex.Message, ex.InnerException?.StackTrace ?? ex.StackTrace);
}
}
// var handleInfos = HandleEnumeratorWrapper.GetHandleInfos();
// var handleCounts = handleInfos.GroupBy(h => h.Type).OrderBy(g => g.Key);
if (exception == null)
{
// Test succeeded.
Log(ReportSystemHeaps
? "{0,3} failures, {1:F2}/{2:F2}/{3:F1} MB, {4}/{5} handles, {6} sec.\r\n"
: "{0,3} failures, {1:F2}/{3:F1} MB, {4}/{5} handles, {6} sec.\r\n",
FailureCount,
ManagedMemory,
CommittedMemory,
TotalMemory,
LastUserHandleCount + LastGdiHandleCount,
LastTotalHandleCount,
LastTestDuration);
// Log("# Heaps " + string.Join("\t", heapCounts.Select(s => s.ToString())) + Environment.NewLine);
// Log("# Handles " + string.Join("\t", handleCounts.Where(c => c.Count() > 14).Select(c => c.Key + ": " + c.Count())) + Environment.NewLine);
if (crtLeakedBytes > CheckCrtLeaks)
{
Log("!!! {0} CRT-LEAKED {1} bytes\r\n", test.TestMethod.Name, crtLeakedBytes);
}
TeamCityFinishTest(test);
return(true);
}
// Save failure information.
FailureCount++;
if (FailureCounts.ContainsKey(test.TestMethod.Name))
{
FailureCounts[test.TestMethod.Name]++;
}
else
{
FailureCounts[test.TestMethod.Name] = 1;
}
var message = exception.InnerException == null ? exception.Message : exception.InnerException.Message;
var stackTrace = exception.InnerException == null ? exception.StackTrace : exception.InnerException.StackTrace;
var failureInfo = "# " + test.TestMethod.Name + "FAILED:\n" +
message + "\n" +
stackTrace;
if (ErrorCounts.ContainsKey(failureInfo))
{
ErrorCounts[failureInfo]++;
}
else
{
ErrorCounts[failureInfo] = 1;
}
TeamCityFinishTest(test, message + '\n' + stackTrace);
Log("{0,3} failures, {1:F2}/{2:F2}/{3:F1} MB, {4}/{5} handles, {6} sec.\r\n\r\n!!! {7} FAILED\r\n{8}\r\n{9}\r\n!!!\r\n\r\n",
FailureCount,
ManagedMemory,
CommittedMemory,
TotalMemory,
LastUserHandleCount + LastGdiHandleCount,
LastTotalHandleCount,
LastTestDuration,
test.TestMethod.Name,
message,
exception);
return(false);
}
public void Run(TestInfo test, CultureInfo culture, Stopwatch stopwatch = null)
{
// Delete test directory.
int testDirectoryCount = 1;
while (Directory.Exists(_testDir))
{
try
{
// Try delete 4 times to give anti-virus software a chance to finish.
// ReSharper disable AccessToModifiedClosure
TryLoop.Try <IOException>(() => Directory.Delete(_testDir, true), 4);
// ReSharper restore AccessToModifiedClosure
}
catch (Exception e)
{
Console.WriteLine("\n\n" + e.Message);
_testDir = SetTestDir(_testContext, ++testDirectoryCount, Process.GetCurrentProcess());
}
}
// Create test class.
var testObject = Activator.CreateInstance(test.TestClassType);
// Set the TestContext.
if (test.SetTestContext != null)
{
var context = new object[] { _testContext };
test.SetTestContext.Invoke(testObject, context);
}
// Switch to selected culture.
LocalizationHelper.CurrentCulture = culture;
LocalizationHelper.InitThread();
// Run the test and time it.
Exception exception = null;
if (stopwatch != null)
{
stopwatch.Start();
}
long totalLeakedBytes = 0;
try
{
if (test.TestInitialize != null)
{
test.TestInitialize.Invoke(testObject, null);
}
if (pass > 1 || repeatCounter > 1)
{
CrtDebugHeap.Checkpoint();
}
test.TestMethod.Invoke(testObject, null);
if (pass > 1 || repeatCounter > 1)
{
long leakedBytes = CrtDebugHeap.DumpLeaks(true);
totalLeakedBytes += leakedBytes;
}
if (test.TestCleanup != null)
{
test.TestCleanup.Invoke(testObject, null);
}
}
catch (Exception e)
{
exception = e;
}
if (stopwatch != null)
{
stopwatch.Stop();
}
// Restore culture.
Thread.CurrentThread.CurrentCulture = saveCulture;
Thread.CurrentThread.CurrentUICulture = saveUICulture;
var managedMemory = GC.GetTotalMemory(false) / mb;
process.Refresh();
var totalMemory = process.PrivateMemorySize64 / mb;
if (exception == null)
{
// Test succeeded.
info = string.Format(
"{0,3} failures, {1:0.0}/{2:0.0} MB{3}, {4} sec.",
_failureCount,
managedMemory,
totalMemory,
totalLeakedBytes > 0 ? string.Format(" *** LEAKED {0} bytes ***", totalLeakedBytes) : "",
stopwatch.ElapsedMilliseconds / 1000);
Console.WriteLine(info);
log.WriteLine(info);
}
else
{
// Save failure information.
_failureCount++;
failureList[testName]++;
info = testName + " {0} failures ({1:0.##}%)\n" +
exception.InnerException.Message + "\n" +
exception.InnerException.StackTrace;
if (errorList.ContainsKey(info))
{
errorList[info]++;
}
else
{
errorList[info] = 1;
}
Console.WriteLine("*** FAILED {0:0.#}% ***", 100.0 * failureList[testName] / pass);
log.WriteLine("{0,3} failures, {1:0.0}/{2:0.0} MB\n*** failure {3}\n{4}\n{5}\n***",
_failureCount, managedMemory, totalMemory, errorList[info], exception.InnerException.Message,
exception.InnerException.StackTrace);
}
log.Flush();
if (totalLeakedBytes > 0)
{
Trace.WriteLine(string.Format("\n*** {0} leaked ***\n", testName));
}
}
public void Run(TestInfo test, CultureInfo culture, Stopwatch stopwatch = null)
{
// Delete test directory.
int testDirectoryCount = 1;
while (Directory.Exists(_testDir))
{
try
{
// Try delete 4 times to give anti-virus software a chance to finish.
// ReSharper disable AccessToModifiedClosure
TryLoop.Try<IOException>(() => Directory.Delete(_testDir, true), 4);
// ReSharper restore AccessToModifiedClosure
}
catch (Exception e)
{
Console.WriteLine("\n\n" + e.Message);
_testDir = SetTestDir(_testContext, ++testDirectoryCount, Process.GetCurrentProcess());
}
}
// Create test class.
var testObject = Activator.CreateInstance(test.TestClassType);
// Set the TestContext.
if (test.SetTestContext != null)
{
var context = new object[] { _testContext };
test.SetTestContext.Invoke(testObject, context);
}
// Switch to selected culture.
LocalizationHelper.CurrentCulture = culture;
LocalizationHelper.InitThread();
// Run the test and time it.
Exception exception = null;
if (stopwatch != null)
stopwatch.Start();
long totalLeakedBytes = 0;
try
{
if (test.TestInitialize != null)
test.TestInitialize.Invoke(testObject, null);
if (pass > 1 || repeatCounter > 1)
CrtDebugHeap.Checkpoint();
test.TestMethod.Invoke(testObject, null);
if (pass > 1 || repeatCounter > 1)
{
long leakedBytes = CrtDebugHeap.DumpLeaks(true);
totalLeakedBytes += leakedBytes;
}
if (test.TestCleanup != null)
test.TestCleanup.Invoke(testObject, null);
}
catch (Exception e)
{
exception = e;
}
if (stopwatch != null)
stopwatch.Stop();
// Restore culture.
Thread.CurrentThread.CurrentCulture = saveCulture;
Thread.CurrentThread.CurrentUICulture = saveUICulture;
var managedMemory = GC.GetTotalMemory(false) / mb;
process.Refresh();
var totalMemory = process.PrivateMemorySize64 / mb;
if (exception == null)
{
// Test succeeded.
info = string.Format(
"{0,3} failures, {1:0.0}/{2:0.0} MB{3}, {4} sec.",
_failureCount,
managedMemory,
totalMemory,
totalLeakedBytes > 0 ? string.Format(" *** LEAKED {0} bytes ***", totalLeakedBytes) : "",
stopwatch.ElapsedMilliseconds/1000);
Console.WriteLine(info);
log.WriteLine(info);
}
else
{
// Save failure information.
_failureCount++;
failureList[testName]++;
info = testName + " {0} failures ({1:0.##}%)\n" +
exception.InnerException.Message + "\n" +
exception.InnerException.StackTrace;
if (errorList.ContainsKey(info))
{
errorList[info]++;
}
else
{
errorList[info] = 1;
}
Console.WriteLine("*** FAILED {0:0.#}% ***", 100.0*failureList[testName]/pass);
log.WriteLine("{0,3} failures, {1:0.0}/{2:0.0} MB\n*** failure {3}\n{4}\n{5}\n***",
_failureCount, managedMemory, totalMemory, errorList[info], exception.InnerException.Message,
exception.InnerException.StackTrace);
}
log.Flush();
if (totalLeakedBytes > 0)
Trace.WriteLine(string.Format("\n*** {0} leaked ***\n", testName));
}
