/// <summary>
/// Runs the reliability tests. Called from Main with the name of the configuration file we should be using.
/// All code in here runs in our starting app domain.
/// </summary>
/// <param name="testConfig">configuration file to use</param>
/// <returns>100 on sucess, another number on failure.</returns>
public int RunReliabilityTests(string testConfig, bool doReplay)
{
_totalSuccess = true;
try
{
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, "Getting configuration...");
_reliabilityConfig = new ReliabilityConfig(testConfig);
}
catch (ArgumentException e)
{
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, String.Format("Error while getting configuration: {0}", e));
return (-1);
}
catch (FileNotFoundException fe)
{
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, String.Format("Couldn't find configuration file: {0}", fe));
return (-1);
}
// save the current directory
string curDir = Directory.GetCurrentDirectory();
#if !PROJECTK_BUILD
if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BASE_ROOT")))
{
Environment.SetEnvironmentVariable("BASE_ROOT", Environment.CurrentDirectory);
}
#endif
// Enumerator through all the test sets...
foreach (ReliabilityTestSet testSet in _reliabilityConfig)
{
if (testSet.InstallDetours)
{
_detourHelpers = new DetourHelpers();
_detourHelpers.Initialize(testSet);
}
else
{
if (_detourHelpers != null)
{
_detourHelpers.Uninitialize();
}
_detourHelpers = null;
}
// restore the current directory incase a test changed it
Directory.SetCurrentDirectory(curDir);
_logger.WriteToInstrumentationLog(testSet, LoggingLevels.Tests, String.Format("Executing test set: {0}", testSet.FriendlyName));
_testsRunningCount = 0;
_testsRanCount = 0;
_curTestSet = testSet;
_logger.ReportResults = _curTestSet.ReportResults;
#if !PROJECTK_BUILD
if (curTestSet.AppDomainLoaderMode == AppDomainLoaderMode.RoundRobin)
{
// full isoloation & normal are handled by the way we setup
// tests in ReliabilityConfiguration. Round robin needs extra
// logic when we create app domains.
_testDomains = new AppDomain[curTestSet.NumAppDomains];
for (int domain = 0; domain < curTestSet.NumAppDomains; domain++)
{
_testDomains[domain] = AppDomain.CreateDomain("RoundRobinDomain" + domain.ToString());
}
}
#endif
if (_curTestSet.ReportResults)
{
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.SmartDotNet, "Reporting results...");
try
{
#if !PROJECTK_BUILD
resultReporter = new Result();
resultReporter.Credentials = CredentialCache.DefaultCredentials;
resultReporter.Url = curTestSet.ReportResultsTo + "/" + "result.asmx";
WebRequest.DefaultWebProxy = null;
Build buildObj = new Build();
buildObj.Url = curTestSet.ReportResultsTo + "/" + "build.asmx";
buildObj.Credentials = CredentialCache.DefaultCredentials;
Guid langGuid = Guid.Empty;
DataSet ds = buildObj.GetGeneralBuildInfo();
foreach (DataTable myTable in ds.Tables)
{
if (myTable.TableName == "BuildAttributeValue")
{
foreach (DataRow myRow in myTable.Rows)
{
if (myRow["AttributeValueName"].ToString() == "English")
{
langGuid = new Guid(myRow["AttributeValueGuid"].ToString());
}
}
}
}
Guid buildGuid = buildObj.GetBuildGuid(
Environment.GetEnvironmentVariable("WHIDBEY_TREE"), // build lab
Environment.GetEnvironmentVariable("SHORTFLAVOR"), // build flavor, string
Environment.GetEnvironmentVariable("VERSION"), // build version, string
new Guid[] { langGuid }); // build attributes, Guid[]
resultGroupGuid = resultReporter.CreateResultGroupEx(
Guid.NewGuid(),
String.Format("{0} - {1}", Environment.MachineName, DateTime.Now.ToShortDateString()),
curTestSet.BvtCategory,
new Guid[] { buildGuid },
new Guid[] { }, // TODO: fill in environmental attributes
false,
0,
"StressRun");
#endif
}
catch (Exception e)
{
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.SmartDotNet, String.Format("Exception while communicating w/ smart.net server: {0}", e));
#if !PROJECTK_BUILD
resultReporter = null;
#endif
AddFailure("Failed to initialize result reporting", null, -1);
}
}
// we don't log while we're replaying a log file.
if (!doReplay)
{
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Logging, "Opening log file...");
if (!_curTestSet.DisableLogging)
{
_logger.OpenLog(_curTestSet.FriendlyName);
}
_logger.WriteStartupInfo(s_seed);
}
if (testSet.Tests == null)
{
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, "No tests to run in test set");
Console.WriteLine("No tests to run, skipping..\r\n");
// no tests in this test set, skip it.
continue;
}
// step 1: preload all the tests, this does NOT start them.
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, "Preloading tests...");
Console.Write("Loading all tests: ");
bool haveAtLeastOneTest = false;
for (int i = 0; i < testSet.Tests.Length; i++)
{
ReliabilityTest test = testSet.Tests[i];
switch (test.TestStartMode)
{
case TestStartModeEnum.ProcessLoader:
Interlocked.Increment(ref LoadingCount);
//for the process loader we just need
//to fill in some details (such as the full path).
TestPreLoader(test, testSet.DiscoveryPaths);
if (test.TestObject == null)
{
Console.WriteLine("Test does not exist: {0}", test);
AddFailure("Test does not exist - disabling.", test, -1);
}
else
{
haveAtLeastOneTest = true;
}
break;
case TestStartModeEnum.AppDomainLoader:
#if PROJECTK_BUILD
Console.WriteLine("Appdomain mode is NOT supported for ProjectK");
#else
// for the app domain loader we create the
// app domains here. This is kinda slow so we
// do it all in parallel.
try
{
if (curTestSet.AppDomainLoaderMode != AppDomainLoaderMode.Lazy)
{
Interlocked.Increment(ref LoadingCount);
test.AppDomainIndex = i % curTestSet.NumAppDomains; // only used for roudn robin scheduling.
TestPreLoaderDelegate loadTestDelegate = new TestPreLoaderDelegate(this.TestPreLoader);
loadTestDelegate.BeginInvoke(test, testSet.DiscoveryPaths, null, null);
}
haveAtLeastOneTest = true;
}
catch { }
#endif
break;
}
Console.Write(".");
}
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, "Finished Preloading tests...");
if (!haveAtLeastOneTest)
{
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, "No tests to execute");
AddFailure("No tests exist!", null, -1);
Console.WriteLine("I have no tests to run!");
continue;
}
while (LoadingCount != 0)
{
int tmp = LoadingCount;
Console.Write("{0,4}\b\b\b\b", tmp);
Thread.Sleep(1000);
}
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, "All tests loaded...");
Console.WriteLine("");
// update the startTime
_startTime = DateTime.Now;
// step 2: start all the tests & run them until we're done.
// if we're in replay mode we'll replay the start order from the log.
if (doReplay)
{
Console.WriteLine("Replaying from log file {0}.log", _curTestSet.FriendlyName);
ExecuteFromLog("Logs\\" + _curTestSet.FriendlyName + ".log");
}
else
{
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, "Beginning test run...");
#if !PROJECTK_BUILD
SetupGeneralUnload();
#endif
TestStarter();
_logger.CloseLog();
}
if ((testSet.PercentPassIsPass != -1 && ((_failCount * 100) / _testsRanCount) < (100 - testSet.PercentPassIsPass)))
{
Console.WriteLine("Some tests failed, but below the fail percent ({0} ran, {1} failed, perecent={2})", _testsRanCount, _failCount, testSet.PercentPassIsPass);
_totalSuccess = true;
}
}
if (_detourHelpers != null)
{
_detourHelpers.Uninitialize();
}
if (_totalSuccess)
{
Console.WriteLine("All tests passed");
return (100);
}
return (99);
}
