// Run all test passes.
private static bool RunTestPasses(
List <TestInfo> testList,
List <TestInfo> unfilteredTestList,
CommandLineArgs commandLineArgs,
StreamWriter log,
long loopCount,
long repeat,
bool profiling = false)
{
bool buildMode = commandLineArgs.ArgAsBool("buildcheck");
bool randomOrder = commandLineArgs.ArgAsBool("random");
bool demoMode = commandLineArgs.ArgAsBool("demo");
bool offscreen = commandLineArgs.ArgAsBool("offscreen");
bool internet = commandLineArgs.ArgAsBool("internet");
bool perftests = commandLineArgs.ArgAsBool("perftests");
bool addsmallmoleculenodes = commandLineArgs.ArgAsBool("testsmallmolecules"); // Add the magic small molecule test node to every document?
bool runsmallmoleculeversions = commandLineArgs.ArgAsBool("runsmallmoleculeversions"); // Run the various tests that are versions of other tests with the document completely converted to small molecules?
bool useVendorReaders = commandLineArgs.ArgAsBool("vendors");
bool showStatus = commandLineArgs.ArgAsBool("status");
bool showFormNames = commandLineArgs.ArgAsBool("showformnames");
bool showMatchingPages = commandLineArgs.ArgAsBool("showpages");
bool qualityMode = commandLineArgs.ArgAsBool("quality");
bool pass0 = commandLineArgs.ArgAsBool("pass0");
bool pass1 = commandLineArgs.ArgAsBool("pass1");
int timeoutMultiplier = (int)commandLineArgs.ArgAsLong("multi");
int pauseSeconds = (int)commandLineArgs.ArgAsLong("pause");
var formList = commandLineArgs.ArgAsString("form");
var pauseDialogs = (string.IsNullOrEmpty(formList)) ? null : formList.Split(',');
var results = commandLineArgs.ArgAsString("results");
var maxSecondsPerTest = commandLineArgs.ArgAsDouble("maxsecondspertest");
// If we haven't been told to run perf tests, remove any from the list
// which may have shown up by default
if (!perftests)
{
for (var t = testList.Count; t-- > 0;)
{
if (testList[t].IsPerfTest)
{
testList.RemoveAt(t);
}
}
for (var ut = unfilteredTestList.Count; ut-- > 0;)
{
if (unfilteredTestList[ut].IsPerfTest)
{
unfilteredTestList.RemoveAt(ut);
}
}
}
// Even if we have been told to run perftests, if none are in the list
// then make sure we don't chat about perf tests in the log
perftests &= testList.Any(t => t.IsPerfTest);
if (buildMode)
{
randomOrder = false;
demoMode = false;
offscreen = true;
useVendorReaders = true;
showStatus = false;
qualityMode = false;
pauseSeconds = 0;
}
var runTests = new RunTests(
demoMode, buildMode, offscreen, internet, showStatus, perftests, addsmallmoleculenodes,
runsmallmoleculeversions,
pauseDialogs, pauseSeconds, useVendorReaders, timeoutMultiplier,
results, log);
if (commandLineArgs.ArgAsBool("clipboardcheck"))
{
runTests.TestContext.Properties["ClipboardCheck"] = "TestRunner clipboard check";
Console.WriteLine("Checking clipboard use for {0} tests...\n", testList.Count);
loopCount = 1;
randomOrder = false;
}
else
{
if (!randomOrder && perftests)
{
runTests.Log("Perf tests will run last, for maximum overall test coverage.\r\n");
}
runTests.Log("Running {0}{1} tests{2}{3}...\r\n",
testList.Count,
testList.Count < unfilteredTestList.Count ? "/" + unfilteredTestList.Count : "",
(loopCount <= 0) ? " forever" : (loopCount == 1) ? "" : " in " + loopCount + " loops",
(repeat <= 1) ? "" : ", repeated " + repeat + " times each per language");
}
// Get list of languages
var languages = buildMode
? new[] { "en" }
: commandLineArgs.ArgAsString("language").Split(',');
if (showFormNames)
{
runTests.Skyline.Set("ShowFormNames", true);
}
if (showMatchingPages)
{
runTests.Skyline.Set("ShowMatchingPages", true);
}
var executingDirectory = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
var qualityLanguages = new FindLanguages(executingDirectory, "en", "fr").Enumerate().ToArray();
var removeList = new List <TestInfo>();
// Pass 0: Test an interesting collection of edge cases:
// French number format,
// No vendor readers,
// No internet access,
// Old reports
if (pass0)
{
runTests.Log("\r\n");
runTests.Log("# Pass 0: Run with French number format, no vendor readers, no internet access, old reports.\r\n");
runTests.Language = new CultureInfo("fr");
runTests.Skyline.Set("NoVendorReaders", true);
runTests.AccessInternet = false;
runTests.LiveReports = false;
runTests.RunPerfTests = false;
runTests.AddSmallMoleculeNodes = false;
runTests.CheckCrtLeaks = CrtLeakThreshold;
bool warnedPass0PerfTest = false;
for (int testNumber = 0; testNumber < testList.Count; testNumber++)
{
var test = testList[testNumber];
if (test.IsPerfTest)
{
// These are largely about vendor and/or internet performance, so not worth doing in pass 0
if (!warnedPass0PerfTest)
{
warnedPass0PerfTest = true;
runTests.Log("# Skipping perf tests for pass 0.\r\n");
}
continue;
}
if (!runTests.Run(test, 0, testNumber))
{
removeList.Add(test);
}
}
runTests.Skyline.Set("NoVendorReaders", false);
runTests.AccessInternet = internet;
runTests.LiveReports = true;
runTests.RunPerfTests = perftests;
runTests.AddSmallMoleculeNodes = addsmallmoleculenodes;
runTests.CheckCrtLeaks = 0;
foreach (var removeTest in removeList)
{
testList.Remove(removeTest);
}
removeList.Clear();
}
// Pass 1: Look for cumulative leaks when test is run multiple times.
if (pass1)
{
runTests.Log("\r\n");
runTests.Log("# Pass 1: Run tests multiple times to detect memory leaks.\r\n");
bool warnedPass1PerfTest = false;
for (int testNumber = 0; testNumber < testList.Count; testNumber++)
{
var test = testList[testNumber];
bool failed = false;
if (test.IsPerfTest)
{
// These are generally too lengthy to run multiple times, so not a good fit for pass 1
if (!warnedPass1PerfTest)
{
warnedPass1PerfTest = true;
runTests.Log("# Skipping perf tests for pass 1 leak checks.\r\n");
}
continue;
}
// Warm up memory by running the test in each language.
for (int i = 0; i < qualityLanguages.Length; i++)
{
runTests.Language = new CultureInfo(qualityLanguages[i]);
if (!runTests.Run(test, 1, testNumber))
{
failed = true;
removeList.Add(test);
break;
}
}
if (failed)
{
continue;
}
// Run test repeatedly until we can confidently assess the leak status.
double slope = 0;
var memoryPoints = new List <double> {
runTests.TotalMemoryBytes
};
for (int i = 0; i < LeakCheckIterations; i++)
{
// Run the test in the next language.
runTests.Language =
new CultureInfo(qualityLanguages[i % qualityLanguages.Length]);
if (!runTests.Run(test, 1, testNumber))
{
failed = true;
removeList.Add(test);
break;
}
// Run linear regression on memory size samples.
var memoryBytes = runTests.TotalMemoryBytes;
memoryPoints.Add(memoryBytes);
if (memoryPoints.Count < 8)
{
continue;
}
// Stop if the leak magnitude is below our threshold.
slope = CalculateSlope(memoryPoints);
if (slope < LeakThreshold)
{
break;
}
memoryPoints.RemoveAt(0);
}
if (failed)
{
continue;
}
if (slope >= LeakThreshold)
{
runTests.Log("!!! {0} LEAKED {1} bytes\r\n", test.TestMethod.Name, Math.Floor(slope));
removeList.Add(test);
}
}
foreach (var removeTest in removeList)
{
testList.Remove(removeTest);
}
removeList.Clear();
}
if (qualityMode)
{
languages = qualityLanguages;
}
// Run all test passes.
int pass = 1;
int passEnd = pass + (int)loopCount;
if (pass0 || pass1)
{
pass++;
passEnd++;
}
if (loopCount <= 0)
{
passEnd = int.MaxValue;
}
if (pass == 2 && pass < passEnd && testList.Count > 0)
{
runTests.Log("\r\n");
runTests.Log("# Pass 2+: Run tests in each selected language.\r\n");
}
int perfPass = pass; // We'll run perf tests just once per language, and only in one language (french) if english and french (along with any others) are both enabled
bool needsPerfTestPass2Warning = testList.Any(t => t.IsPerfTest); // No perf tests, no warning
bool flip = true;
var perfTestsFrenchOnly = perftests && languages.Any(l => l.StartsWith("en")) && languages.Any(l => l.StartsWith("fr"));
for (; pass < passEnd; pass++)
{
if (testList.Count == 0)
{
break;
}
// Run each test in this test pass.
var testPass = randomOrder ? testList.RandomOrder().ToList() : testList;
for (int testNumber = 0; testNumber < testPass.Count; testNumber++)
{
var test = testPass[testNumber];
// Perf Tests are generally too lengthy to run multiple times (but non-english format check is useful)
var languagesThisTest = (test.IsPerfTest && perfTestsFrenchOnly) ? new[] { "fr" } : languages;
if (perfTestsFrenchOnly && needsPerfTestPass2Warning)
{
// NB the phrase "# Perf tests" in a log is a key for SkylineNightly to post to a different URL - so don't mess with this.
runTests.Log("# Perf tests will be run only once, and only in French. To run perf tests in other languages, enable all but English.\r\n");
needsPerfTestPass2Warning = false;
}
// Run once (or repeat times) for each language.
for (int i = 0; i < languagesThisTest.Length; i++)
{
runTests.Language = new CultureInfo(languagesThisTest[i]);
var stopWatch = new Stopwatch();
stopWatch.Start(); // Limit the repeats in case of very long tests
for (int repeatCounter = 1; repeatCounter <= repeat; repeatCounter++)
{
if (test.IsPerfTest && ((pass > perfPass) || (repeatCounter > 1)))
{
// Perf Tests are generally too lengthy to run multiple times (but per-language check is useful)
if (needsPerfTestPass2Warning)
{
// NB the phrase "# Perf tests" in a log is a key for SkylineNightly to post to a different URL - so don't mess with this.
runTests.Log("# Perf tests will be run only once per language.\r\n");
needsPerfTestPass2Warning = false;
}
break;
}
if (!runTests.Run(test, pass, testNumber))
{
removeList.Add(test);
i = languages.Length - 1; // Don't run other languages.
break;
}
if (maxSecondsPerTest > 0)
{
var maxSecondsPerTestPerLanguage = maxSecondsPerTest / languagesThisTest.Length; // We'd like no more than 5 minutes per test across all languages when doing stess tests
if (stopWatch.Elapsed.TotalSeconds > maxSecondsPerTestPerLanguage && repeatCounter <= repeat - 1)
{
runTests.Log(string.Format("# Breaking repeat test at count {0} of requested {1} (at {2} minutes), to allow other tests and languages to run.\r\n", repeatCounter, repeat, stopWatch.Elapsed.TotalMinutes));
break;
}
}
}
if (profiling)
{
break;
}
}
}
foreach (var removeTest in removeList)
{
testList.Remove(removeTest);
}
removeList.Clear();
runTests.AddSmallMoleculeNodes = addsmallmoleculenodes && (flip = !flip); // Do this in every other pass, so we get it both ways
}
return(runTests.FailureCount == 0);
}