/// <summary>
/// Writes a pre-command failure into the log
/// </summary>
public void WritePreCommandFailure(ReliabilityTest test, string command, string commandType)
{
WriteToLog(String.Format(" <PreCommandFailure Command=\"\" CommandType=\"{1}\" TestId=\"{2}\"",
command,
commandType,
test.RefOrID));
}
/// <summary>
/// Writes a test pass into the log.
/// </summary>
public void WriteTestPass(ReliabilityTest test, string message)
{
string testName = (test == null) ? "Harness" : test.RefOrID;
WriteToLog(String.Format(" <TestPass DateTime=\"{0}\" TestId=\"{1}\" Description=\"{2}\" />\r\n",
DateTime.Now,
testName,
message));
}
/// <summary>
/// Writes the detection of a race into the log
/// </summary>
public void WriteTestRace(ReliabilityTest test)
{
WriteToLog(String.Format("<TestRace DateTime=\"{0}\" TestId=\"\"/>\r\n", DateTime.Now, test.RefOrID));
}
/// <summary>
/// Report that a test has successfully passed
/// </summary>
/// <param name="passMsg">an additional message about the test passing</param>
/// <param name="testRefOrID">the test which passed</param>
private void AddSuccess(string passMsg, ReliabilityTest test, int returnCode)
{
if (_curTestSet.ReportResults && test.Guid != Guid.Empty)
{
// smart.net test result reporting
// smart.net test result reporting
try
{
#if !PROJECTK_BUILD
lock (resultReporter)
{
resultReporter.InsertNewTestResult(
Guid.Empty, // test result (guid)
resultGroupGuid, // result group (guid)
test.Guid, // test command line (guid)
Guid.Empty, // clover bug (guid)
curTestSet.BvtCategory, // bvt category (guid)
Guid.Empty, // machine image (guid)
false, // is failure (bool)
returnCode, // return code (int)
test.StartTime, // start time (DateTime)
DateTime.Now, // end time (DateTime)
String.Empty, // automation comment (string)
passMsg); // comment (string)
}
#endif
}
catch (Exception e)
{
Console.WriteLine("Unable to save test result: {0}", e);
}
}
// log the failure
_logger.WriteTestPass(test, passMsg);
}
/*
/// <summary>
/// This method will send a failure message to the test owner that their test has failed.
/// </summary>
/// <param name="testCase">the test case which failed</param>
/// <param name="returnCode">return code of the test, -1 for none provided</param>
void SendFailMail(ReliabilityTest testCase, string message, string subject, string to, string body)
{
// we only want to send fail mails once / test / run, so we mark a failed test
// and won't send any additional e-mails once it's failed.
if (testCase == null || !testCase.HasFailed)
{
if (testCase != null)
{
testCase.HasFailed = true;
}
try
{
#pragma warning disable 618
MailMessage mail = new MailMessage();
#pragma warning restore
if (subject != null)
{
mail.Subject = subject;
}
else
{
mail.Subject = "ACTION REQUIRED::Follow up on stress failures";
}
mail.From = "*****@*****.**";
if (to == null)
{
if (testCase != null && testCase.TestOwner != null)
{
string[] failMailReceivers = testCase.TestOwner.Split(new char[] { ';' });
//convert aliases into full e-mail addresses
foreach (string failReceiver in failMailReceivers)
{
if (failReceiver.IndexOf("@") != -1)
{
mail.To = failReceiver;
}
else
{
mail.To = String.Format("{0}@microsoft.com", failReceiver);
}
}
}
else
{
mail.To = "*****@*****.**";
}
}
else
{
mail.To = to;
}
if (curTestSet.CCFailMail != null)
{
mail.Cc = curTestSet.CCFailMail;
}
#pragma warning disable 0618
mail.BodyFormat = MailFormat.Html;
mail.Priority = MailPriority.High;
#pragma warning restore
if (body == null)
{
mail.Body = String.Format(@"
<HTML><BODY><H2>Please investigate your test failures ASAP</H2>
<table border=1 cellspacing=0>
<tr><td bgcolor=#cccccc>Computer Name:</td><td> {0}</td></tr>
<tr><td bgcolor=#cccccc>Test :</td><td> {1} {2}</td></tr>
<tr><td bgcolor=#cccccc>Comments :</td><td> {3}</td></tr>
</table>
<P>If you are listed on the To: line, you have test failures to investigate.
<p>For all failures please find the machine listed above on the <a href=""http://urtframeworks/stress/stressdetails.aspx?team=CLR"">CLR Stress Details Web Page</a> and open a tracking bug if one has not already been created for this stress run.
<p>If this is a product failure please e-mail the
<a href=""mailto:corqrd"">CLR Quick Response Dev Team</a> with the failure information and tracking bug number. The QRT will then open a product bug if appropriate and resolve the tracking bug as a duplicate.
<p>If this is a test failure please open a tracking bug via the CLR Stress Details web page and assign if to yourself. Resolve the bug once you have fixed the test issue.
<p>If this is a stress harness issue please contact <a href=""mailto:timme;dinov"">the stress developers</a>.
Thanks for contributing to CLR Stress!
</P></BODY></HTML>", Environment.MachineName, testCase == null ? "None" : testCase.Assembly, testCase == null ? "None" : testCase.Arguments, message);
}
else
{
mail.Body = body;
}
#pragma warning disable 0618
SmtpMail.SmtpServer = "smarthost";
SmtpMail.Send(mail);
#pragma warning restore
}
catch (Exception e)
{
Console.WriteLine("Error sending fail mail: {0}", e);
}
}
}
*/
/// <summary>
/// Add a failure to the failure log.
/// </summary>
/// <param name="failMsg">the message to add (the reason the failure happened)</param>
/// <param name="test">the test that failed</param>
private void AddFailure(string failMsg, ReliabilityTest test, int returnCode)
{
// bump the fail count
Interlocked.Increment(ref _failCount);
// log the failure
_logger.WriteTestFail(test, failMsg);
// report results back to harnesses / urt frameworks.
_totalSuccess = false;
if (test != null)
{
try
{
// Record the failure to the database
string arguments = String.Format("//b //nologo %SCRIPTSDIR%\\record.js -i %STRESSID% -a LOG_FAILED_TEST -k \"FAILED {0}\"", test.RefOrID);
ProcessStartInfo psi = new ProcessStartInfo("cscript.exe", Environment.ExpandEnvironmentVariables(arguments));
psi.UseShellExecute = false;
psi.RedirectStandardOutput = true;
Process p = Process.Start(psi);
p.StandardOutput.ReadToEnd();
p.WaitForExit();
if (p.ExitCode != 0)
{
Console.WriteLine("//b //nologo record.js -i %STRESSID% -a LOG_FAILED_TEST -k \"{0}\"", test.RefOrID);
}
p.Dispose();
}
catch
{
Console.WriteLine("Exception while trying to log a test failure to the custom table.");
}
#if !PROJECTK_BUILD
if (curTestSet.ReportResults && test.Guid != Guid.Empty && resultReporter != null)
{
lock (resultReporter)
{
try
{
// smart.net test result reporting
resultReporter.InsertNewTestResult(
Guid.Empty, // test result (guid)
resultGroupGuid, // result group (guid)
test.Guid, // test command line (guid)
Guid.Empty, // clover bug (guid)
curTestSet.BvtCategory, // bvt category (guid)
Guid.Empty, // machine image (guid)
true, // is failure (bool)
returnCode, // return code (int)
test.StartTime, // start time (DateTime)
DateTime.Now, // end time (DateTime)
String.Empty, // automation comment (string)
failMsg); // comment (string)
}
catch (Exception e)
{
Console.WriteLine("Exception while storing results: {0}", e);
}
}
}
#endif
}
try
{
if (_curTestSet.ReportResults && File.Exists(Environment.ExpandEnvironmentVariables("%SCRIPTSDIR%\\record.js")))
{
string arguments;
if (test == null)
{
arguments = String.Format("//b //nologo %SCRIPTSDIR%\\record.js -i %STRESSID% -a ADD_CUSTOM -s RUNNING -k FAIL{0:000} -v \"(non test failure)\"", _reportedFailCnt++);
}
else
{
arguments = String.Format("//b //nologo %SCRIPTSDIR%\\record.js -i %STRESSID% -a ADD_CUSTOM -s RUNNING -k FAIL{0:000} -v \"{1} ({2} ReturnCode={3})\"", _reportedFailCnt++, test.RefOrID, test.TestOwner, returnCode);
}
ProcessStartInfo psi = new ProcessStartInfo("cscript.exe", Environment.ExpandEnvironmentVariables(arguments));
psi.UseShellExecute = false;
psi.RedirectStandardOutput = true;
Process p = Process.Start(psi);
p.StandardOutput.ReadToEnd();
p.WaitForExit();
if (p.ExitCode != 0)
{
Console.WriteLine("cscript.exe " + Environment.ExpandEnvironmentVariables("//b //nologo %SCRIPTSDIR%\\record.js -i %STRESSID% -a UPDATE_RECORD -s RUNNING"));
Console.WriteLine("WARNING: Status update did not return success! {0}", p.ExitCode);
}
p.Dispose();
}
else
{
_logger.LogNoResultReporter(_curTestSet.ReportResults);
}
}
catch (Exception e)
{
Console.WriteLine("WARNING: Status update did not return success (exception thrown {0})!", e);
}
if (test == null)
{
//SendFailMail(test,failMsg);
}
}
/// <summary>
/// This method will send a failure message to the test owner that their test has failed.
/// </summary>
/// <param name="testCase">the test case which failed</param>
/// <param name="returnCode">return code of the test, -1 for none provided</param>
private void SendFailMail(ReliabilityTest testCase, string message)
{
//SendFailMail(testCase, message, null, null, null);
}
/// <summary>
/// Unloads a test & the tests associated app domain.
/// </summary>
/// <param name="test"></param>
void UnloadTestAndAD(ReliabilityTest test)
{
if (test.TestObject is ISingleReliabilityTest)
{
((ISingleReliabilityTest)test.TestObject).Unregister();
}
else if (test.TestObject is IMultipleReliabilityTest)
{
((IMultipleReliabilityTest)test.TestObject).Unregister();
}
AppDomain.Unload(test.AppDomain);
}
/// <summary>
/// Preloads a test for a process, this just sets the test object to the appropriate path.
/// </summary>
/// <param name="test"></param>
/// <param name="paths"></param>
private void TestPreLoader_Process(ReliabilityTest test, string[] paths)
{
Console.WriteLine("Preloading for process mode");
string realpath = ReliabilityConfig.ConvertPotentiallyRelativeFilenameToFullPath(test.BasePath, test.Assembly);
Debug.Assert(test.TestObject == null);
if (File.Exists(realpath))
{
test.TestObject = realpath;
}
else if (File.Exists((string)test.Assembly))
{
test.TestObject = test.Assembly;
}
else
{
foreach (string path in paths)
{
string fullPath = ReliabilityConfig.ConvertPotentiallyRelativeFilenameToFullPath(path, (string)test.Assembly);
if (File.Exists(fullPath))
{
test.TestObject = fullPath;
break;
}
}
}
if (test.TestObject == null)
{
Console.WriteLine("Couldn't find path for {0}", test.Assembly);
}
#if PROJECTK_BUILD
if (test.EntryPointMethod == null)
{
CustomAssemblyResolver resolver = new CustomAssemblyResolver();
// test.Assembly is with the extension. LoadFromAssemblyName needs it without.
string strAssemblyNameWithoutExt = Path.ChangeExtension(test.Assembly, null);
Assembly testAssembly = resolver.LoadFromAssemblyName(new AssemblyName(strAssemblyNameWithoutExt));
Type[] testTypes = AssemblyExtensions.GetTypes(testAssembly);
MethodInfo methodInfo = null;
if (testTypes != null)
{
foreach (Type t in testTypes)
{
methodInfo = t.GetMethod("Main");
if (methodInfo != null)
{
//Console.WriteLine(t.FullName + " contains the entrypoint");
break;
}
}
}
test.EntryPointMethod = methodInfo;
}
#endif
}
/// <summary>
/// Writes the detection of a race into the log
/// </summary>
public void WriteTestRace(ReliabilityTest test)
{
WriteToLog(String.Format("<TestRace DateTime=\"{0}\" TestId=\"\"/>\r\n", DateTime.Now, test.RefOrID));
}
/// <summary>
/// Loads the specified test into a new app domain. Returns true on success and false on failure.
/// </summary>
/// <param name="test">the test to execute</param>
/// <param name="paths">paths where to search for the assembly if it's not found.</param>
/// <returns>true if the test was successfully loaded & executed, false otherwise.</returns>
private void TestPreLoader(ReliabilityTest test, string[] paths)
{
try
{
RunCommands(test.PreCommands, "pre", test);
switch (test.TestStartMode)
{
case TestStartModeEnum.ProcessLoader:
TestPreLoader_Process(test, paths);
break;
case TestStartModeEnum.AppDomainLoader:
#if PROJECTK_BUILD
Console.WriteLine("Appdomain mode is NOT supported for ProjectK");
#else
TestPreLoader_AppDomain(test, paths);
#endif
break;
}
}
catch (Exception e)
{
string msg = String.Format("\r\nBad test ({1} - {3}): {0}\r\n{2}\r\n{4}", test.RefOrID, e.GetType(), waitingText, e.Message, e.StackTrace);
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, msg);
test.ConcurrentCopies = 0;
#if !PROJECTK_BUILD
test.AppDomain = null;
#endif
test.TestLoadFailed = true;
if (_curTestSet.DebugBreakOnBadTest)
{
BadTestDebugBreak(msg);
}
}
Interlocked.Decrement(ref LoadingCount);
}
/// <summary>
/// Runs a list of commands stored in an array list. Logs any failures.
/// </summary>
/// <param name="commands">the array list of commands</param>
/// <param name="commandType">the type of commands (used only for logging)</param>
private void RunCommands(List<string> commands, string commandType, ReliabilityTest test)
{
if (commands != null)
{
for (int i = 0; i < commands.Count; i++)
{
ProcessStartInfo pi = null;
Process p = null;
try
{
Debug.Assert(commands[i] is string, "Non-string in command list!");
string torun = ((string)commands[i]).Replace("__ASSEMBLY__", test.Assembly);
string args = null;
if (torun.IndexOf(' ') != -1)
{
args = torun.Substring(torun.IndexOf(' ') + 1);
torun = torun.Substring(0, torun.IndexOf(' '));
}
pi = new ProcessStartInfo(torun);
if (test.BasePath != String.Empty)
{
pi.WorkingDirectory = test.BasePath;
}
if (args != null)
{
pi.Arguments = args;
}
pi.UseShellExecute = true;
p = Process.Start(pi);
p.WaitForExit();
}
catch
{
_logger.WritePreCommandFailure(test, (string)commands[i], commandType);
}
finally
{
if (null != p)
{
p.Dispose();
}
}
}
}
}
/// <summary>
/// Called after a test has finished executing.
/// </summary>
/// <param name="test"></param>
public void SignalTestFinished(ReliabilityTest test)
{
Interlocked.Decrement(ref _testsRunningCount);
_testDone.Set(); // we signal the event before we do the lock() below because the lock could throw due to OOM.
test.TestStopped();
}
/// <summary>
/// Starts the test passed. The test should already be loaded into an app domain.
/// </summary>
/// <param name="test">The test to run.</param>
private void StartTest(ReliabilityTest test)
{
#if PROJECTK_BUILD
Interlocked.Increment(ref _testsRunningCount);
test.TestStarted();
StartTestWorker(test);
#else
try
{
if (curTestSet.AppDomainLoaderMode == AppDomainLoaderMode.Lazy)
{
Console.WriteLine("Test Loading: {0} run #{1}", test.RefOrID, test.RunCount);
TestPreLoader(test, curTestSet.DiscoveryPaths);
}
// Any test failed to load during the preload step should be removed. Need to take a closer look.
// This is to fix Dev10 Bug 552621.
if (test.TestLoadFailed)
{
logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.Tests, "Test failed to load.");
return;
}
Thread newThread = new Thread(new ParameterizedThreadStart(this.StartTestWorker));
// check the thread requirements and set appropriately.
switch ((test.TestAttrs & TestAttributes.RequiresThread))
{
case TestAttributes.RequiresSTAThread:
newThread.SetApartmentState(ApartmentState.STA);
break;
case TestAttributes.RequiresMTAThread:
newThread.SetApartmentState(ApartmentState.MTA);
break;
case TestAttributes.RequiresUnknownThread:
// no attribute specified... ignore.
break;
}
newThread.Name = test.RefOrID;
Interlocked.Increment(ref testsRunningCount);
test.TestStarted();
logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.TestStarter, String.Format("RF.StartTest, RTs({0}) - Instances of this test: {1} - New Test:{2}", testsRunningCount, test.RunningCount, test.RefOrID));
newThread.Start(test);
}
catch (OutOfMemoryException e)
{
HandleOom(e, "StartTest");
}
#endif
}
/// <summary>
/// Writes a test pass into the log.
/// </summary>
public void WriteTestPass(ReliabilityTest test, string message)
{
string testName = (test == null) ? "Harness" : test.RefOrID;
WriteToLog(String.Format(" <TestPass DateTime=\"{0}\" TestId=\"{1}\" Description=\"{2}\" />\r\n",
DateTime.Now,
testName,
message));
}
/// <summary>
/// Writes a pre-command failure into the log
/// </summary>
public void WritePreCommandFailure(ReliabilityTest test, string command, string commandType)
{
WriteToLog(String.Format(" <PreCommandFailure Command=\"\" CommandType=\"{1}\" TestId=\"{2}\"",
command,
commandType,
test.RefOrID));
}
/// <summary>
/// Pre-loads a test into the correct app domain for the current loader mode.
/// </summary>
/// <param name="test"></param>
/// <param name="paths"></param>
void TestPreLoader_AppDomain(ReliabilityTest test, string[] paths)
{
AppDomain ad = null;
try
{
if (curTestSet.AppDomainLoaderMode != AppDomainLoaderMode.RoundRobin || test.CustomAction == CustomActionType.LegacySecurityPolicy)
{
string appDomainName = AppDomain.CurrentDomain.FriendlyName + "_" + "TestDomain_" + test.Assembly + "_" + Guid.NewGuid().ToString();
logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.AppDomain, "Creating app domain: " + appDomainName + " for " + test.Index.ToString());
AppDomainSetup ads = new AppDomainSetup();
Evidence ev = AppDomain.CurrentDomain.Evidence;
if (test.CustomAction == CustomActionType.LegacySecurityPolicy)
{
ads.SetCompatibilitySwitches(new string[] { "NetFx40_LegacySecurityPolicy" });
ev = new Evidence(new EvidenceBase[] { new Zone(System.Security.SecurityZone.MyComputer) }, null);
}
// Set the probing scope for assemblies to %BVT_ROOT%. The default is %BVT_ROOT%\Stress\CLRCore,
// which causes some tests to fail because their assemblies are out of scope.
ads.ApplicationBase = "file:///" + Environment.GetEnvironmentVariable("BVT_ROOT").Replace(@"\", "/");
ads.PrivateBinPath = "file:///" + Environment.GetEnvironmentVariable("BASE_ROOT").Replace(@"\", "/");
ad = AppDomain.CreateDomain(appDomainName, ev, ads);
}
else
{
ad = _testDomains[test.AppDomainIndex];
}
AssemblyName an = new AssemblyName();
Object ourObj = null;
test.AppDomain = ad;
object obj = ad.CreateInstanceAndUnwrap(Assembly.GetExecutingAssembly().FullName, typeof(LoaderClass).FullName);
LoaderClass lfc = obj as LoaderClass;
if (test.SuppressConsoleOutput)
lfc.SuppressConsole();
if (test.Assembly.ToLower().IndexOf(".exe") == -1 && test.Assembly.ToLower().IndexOf(".dll") == -1) // must be a simple name or fullname...
{
lfc.Load(test.Assembly, paths, this);
}
else // has an executable extension, must be in local directory.
{
lfc.LoadFrom(test.BasePath + test.Assembly, paths, this);
}
// check and see if this test is marked as requiring STA. We only do
// the check once, and then we set the STA/MTA/Unknown bit on the test attributes
// to avoid doing reflection every time we start the test.
if ((test.TestAttrs & TestAttributes.RequiresThread) == TestAttributes.None)
{
ApartmentState state = lfc.CheckMainForThreadType();
switch (state)
{
case ApartmentState.STA:
test.TestAttrs |= TestAttributes.RequiresSTAThread;
break;
case ApartmentState.MTA:
test.TestAttrs |= TestAttributes.RequiresMTAThread;
break;
case ApartmentState.Unknown:
test.TestAttrs |= TestAttributes.RequiresUnknownThread;
break;
}
}
ourObj = lfc.GetTest();
// and now call the register method on the type if it's one of our supported test types.
if (ourObj is ISingleReliabilityTest)
{
((ISingleReliabilityTest)ourObj).Register();
}
else if (ourObj is IMultipleReliabilityTest)
{
((IMultipleReliabilityTest)ourObj).Register();
}
else if (!(ourObj is string)) // we were unable to find a test here - a string is an executable filename.
{
Interlocked.Decrement(ref LoadingCount);
return;
}
test.TestObject = ourObj;
test.MyLoader = lfc;
}
catch (Exception)
{
// if we took an exception while loading the test, but we still have an app domain
// we don't want to leak the app domain.
if (ad != null)
{
test.AppDomain = null;
AppDomain.Unload(ad);
}
throw;
}
}
/// <summary>
/// Given a test configfile we find the tests that we actually want to run.
/// </summary>
private void GetTestsToRun(string testConfig)
{
int totalDepth = 0; // used for debugging mode so we can keep proper indentation.
ArrayList foundTests = new ArrayList(); // the array of tests we've found.
ArrayList discoveryPaths = new ArrayList(); // the array of discovery paths we've found.
Stack xmlFileStack = new Stack(); // this stack keeps track of our include files.
Stack testLevelStack = new Stack();
try
{
#if PROJECTK_BUILD
FileStream fs = new FileStream(testConfig, FileMode.Open, FileAccess.Read, FileShare.Read);
xmlFileStack.Push(XmlReader.Create(fs));
#else
xmlFileStack.Push(new XmlTextReader(testConfig));
#endif
}
catch (FileNotFoundException e)
{
Console.WriteLine("Could not open config file: {0}", testConfig);
throw e;
}
do
{
#if PROJECTK_BUILD
XmlReader currentXML = (XmlReader)xmlFileStack.Pop();
#else
XmlTextReader currentXML = (XmlTextReader)xmlFileStack.Pop();
#endif
totalDepth -= currentXML.Depth;
if (currentXML.Depth != 0)
{
IndentToDepth(totalDepth + currentXML.Depth - 1); // -1 because we haven't done a .Read on the includes tag yet.
XmlDebugOutLine("</" + configInclude + ">");
}
while (currentXML.Read())
{
switch (currentXML.NodeType)
{
case XmlNodeType.Element:
bool isEmpty = currentXML.IsEmptyElement;
IndentToDepth(totalDepth + currentXML.Depth);
XmlDebugOut("<" + currentXML.Name);
switch (currentXML.Name)
{
case configInclude: // user included a file in this file.
string filename = null;
bool skipInclude = false;
while (currentXML.MoveToNextAttribute())
{
XmlDebugOut(" " + currentXML.Name + "=\"" + currentXML.Value + "\"");
switch (currentXML.Name)
{
case configIncludeFilename:
filename = currentXML.Value;
break;
case debugConfigIncludeInlined: // so we can consume the XML we spit out in debug mode-
// we ignore this include tag if it's been inlined.
if (currentXML.Value.ToLower() == "true" || currentXML.Value == "1")
{
skipInclude = true;
}
break;
default:
throw new Exception("Unknown attribute on include tag!");
}
}
if (skipInclude)
{
XmlDebugOutLine(">");
continue;
}
XmlDebugOut(" " + debugConfigIncludeInlined + "=\"true\">\r\n");
if (filename == null)
{
throw new ArgumentException("Type or Filename not set on include file! Both attributes must be set to properly include a file.");
}
xmlFileStack.Push(currentXML); // save our current file.
totalDepth += currentXML.Depth;
filename = ConvertPotentiallyRelativeFilenameToFullPath(stripFilenameFromPath(currentXML.BaseURI), filename);
try
{
#if PROJECTK_BUILD
currentXML = XmlReader.Create(filename);
#else
currentXML = new XmlTextReader(filename);
#endif
}
catch (FileNotFoundException e)
{
Console.WriteLine("Could not open included config file: {0}", filename);
throw e;
}
continue;
case configIncludes:
if (isEmpty)
{
XmlDebugOut("/>\r\n");
}
else
{
XmlDebugOut(">\r\n");
}
continue; // note: we never push or pop includes off of our stack.
case configHost:
if (testLevelStack.Count == 0) // we'll skip this tag when it shows up in an included file.
{
testLevelStack.Push(configHost);
while (currentXML.MoveToNextAttribute())
{
switch (currentXML.Name)
{
case "xmlns:xsi":
case "xmlns:xsd":
break;
default:
throw new Exception("Unknown attribute on reliability tag: " + currentXML.Name);
}
}
}
else
{
if (isEmpty)
{
XmlDebugOutLine("/>");
}
else
{
XmlDebugOutLine(">");
}
continue;
}
break;
case concurrentConfigTest:
if (testLevelStack.Count != 0 && (string)testLevelStack.Peek() != configHost)
{
throw new ArgumentException("The test tag can only appear as a child to the reliabilityFramework tag or a top level tag.");
}
// save any info we've gathered about tests into the current test set
if (_curTestSet != null && foundTests != null && foundTests.Count > 0)
{
_curTestSet.Tests = (ReliabilityTest[])foundTests.ToArray(typeof(ReliabilityTest));
_curTestSet.DiscoveryPaths = (string[])discoveryPaths.ToArray(typeof(string));
discoveryPaths.Clear();
foundTests.Clear();
}
testLevelStack.Push(concurrentConfigTest);
_curTestSet = new ReliabilityTestSet();
while (currentXML.MoveToNextAttribute())
{
XmlDebugOut(" " + currentXML.Name + "=\"" + currentXML.Value + "\"");
switch (currentXML.Name)
{
case "maximumTestRuns":
_curTestSet.MaximumLoops = Convert.ToInt32(currentXML.Value);
break;
case "maximumExecutionTime":
string timeValue = currentXML.Value;
_curTestSet.MaximumTime = ConvertTimeValueToTestRunTime(timeValue);
break;
case "id":
_curTestSet.FriendlyName = currentXML.Value;
break;
case "xmlns:xsi":
case "xmlns:xsd":
break;
case configTestMinimumMem:
_curTestSet.MinPercentMem = Convert.ToInt32(currentXML.Value);
break;
case configLoggingLevel:
_curTestSet.LoggingLevel = (LoggingLevels)Convert.ToInt32(currentXML.Value.ToString(), 16);
break;
case configTestMinimumCPUStaggered:
_curTestSet.MinPercentCPUStaggered = currentXML.Value;
break;
case configTestMinimumCPU:
_curTestSet.MinPercentCPU = Convert.ToInt32(currentXML.Value);
break;
case configInstallDetours:
if (currentXML.Value == "true" || currentXML.Value == "1" || currentXML.Value == "yes")
{
_curTestSet.InstallDetours = true;
}
else if (currentXML.Value == "false" || currentXML.Value == "0" || currentXML.Value == "no")
{
_curTestSet.InstallDetours = false;
}
else
{
throw new Exception("Unknown value for result reporting: " + currentXML.Value);
}
break;
case configTestMinimumTests:
_curTestSet.MinTestsRunning = Convert.ToInt32(currentXML.Value);
break;
case RFConfigOptions.RFConfigOptions_Test_MinMaxTestsUseCPUCount:
if (GetTrueFalseOptionValue(currentXML.Value, RFConfigOptions.RFConfigOptions_Test_MinMaxTestsUseCPUCount))
{
int CPUCount = Convert.ToInt32(Environment.GetEnvironmentVariable("NUMBER_OF_PROCESSORS"));
if (CPUCount <= 0)
throw new Exception("Invalid Value when reading NUMBER_OF_PROCESSORS: {0}" + CPUCount);
_curTestSet.MinTestsRunning = CPUCount;
_curTestSet.MaxTestsRunning = (int)(CPUCount * 1.5);
}
break;
case RFConfigOptions.RFConfigOptions_Test_SuppressConsoleOutputFromTests:
_curTestSet.SuppressConsoleOutputFromTests = GetTrueFalseOptionValue(currentXML.Value, RFConfigOptions.RFConfigOptions_Test_SuppressConsoleOutputFromTests);
break;
case RFConfigOptions.RFConfigOptions_Test_DebugBreakOnHang:
_curTestSet.DebugBreakOnTestHang = GetTrueFalseOptionValue(currentXML.Value, RFConfigOptions.RFConfigOptions_Test_DebugBreakOnHang);
break;
case RFConfigOptions.RFConfigOptions_Test_DebugBreakOnBadTest:
_curTestSet.DebugBreakOnBadTest = GetTrueFalseOptionValue(currentXML.Value, RFConfigOptions.RFConfigOptions_Test_DebugBreakOnBadTest);
break;
case RFConfigOptions.RFConfigOptions_Test_DebugBreakOnOutOfMemory:
_curTestSet.DebugBreakOnOutOfMemory = GetTrueFalseOptionValue(currentXML.Value, RFConfigOptions.RFConfigOptions_Test_DebugBreakOnOutOfMemory);
break;
case RFConfigOptions.RFConfigOptions_Test_DebugBreakOnPathTooLong:
_curTestSet.DebugBreakOnPathTooLong = GetTrueFalseOptionValue(currentXML.Value, RFConfigOptions.RFConfigOptions_Test_DebugBreakOnPathTooLong);
break;
case RFConfigOptions.RFConfigOptions_Test_DebugBreakOnMissingTest:
_curTestSet.DebugBreakOnMissingTest = GetTrueFalseOptionValue(currentXML.Value, RFConfigOptions.RFConfigOptions_Test_DebugBreakOnMissingTest);
break;
case configResultReporting:
_curTestSet.ReportResults = GetTrueFalseOptionValue(currentXML.Value, configResultReporting);
break;
case configResultReportingUrl:
_curTestSet.ReportResultsTo = currentXML.Value;
break;
case configResultReportingBvtCategory:
try
{
_curTestSet.BvtCategory = new Guid(currentXML.Value);
}
catch (FormatException)
{
throw new Exception(String.Format("BVT Category Guid {0} is not in the correct form", currentXML.Value));
}
break;
case configTestMaximumTests:
_curTestSet.MaxTestsRunning = Convert.ToInt32(currentXML.Value);
break;
case configTestDisableLogging:
_curTestSet.DisableLogging = GetTrueFalseOptionValue(currentXML.Value, configTestDisableLogging);
break;
case configEnablePerfCounters:
_curTestSet.EnablePerfCounters = GetTrueFalseOptionValue(currentXML.Value, configEnablePerfCounters);
break;
case configDefaultTestStartMode:
switch (currentXML.Value)
{
case configTestStartModeAppDomainLoader:
if (null != _curTestSet.DefaultDebugger || null != _curTestSet.DefaultDebuggerOptions)
{
throw new Exception(String.Format("{0} specified with default debugger or debugger options. If you want a debugger per test please use {1}=\"{2}\" ",
configTestStartModeAppDomainLoader,
configDefaultTestStartMode,
configTestStartModeProcessLoader));
}
_curTestSet.DefaultTestStartMode = TestStartModeEnum.AppDomainLoader;
break;
case configTestStartModeProcessLoader:
_curTestSet.DefaultTestStartMode = TestStartModeEnum.ProcessLoader;
break;
default:
throw new Exception(String.Format("Unknown test starter {0} specified!", currentXML.Value));
}
break;
case configRoundRobinAppDomainCount:
try
{
_curTestSet.NumAppDomains = Convert.ToInt32(currentXML.Value);
if (_curTestSet.NumAppDomains <= 0)
{
throw new Exception("Number of app domains must be greater than zero!");
}
}
catch
{
throw new Exception(String.Format("The value {0} is not an integer", currentXML.Value));
}
break;
case configAppDomainLoaderMode:
switch (currentXML.Value)
{
case configAppDomainLoaderModeFullIsolation:
_curTestSet.AppDomainLoaderMode = AppDomainLoaderMode.FullIsolation;
break;
case configAppDomainLoaderModeNormal:
_curTestSet.AppDomainLoaderMode = AppDomainLoaderMode.Normal;
break;
case configAppDomainLoaderModeRoundRobin:
_curTestSet.AppDomainLoaderMode = AppDomainLoaderMode.RoundRobin;
break;
case configAppDomainLoaderModeLazy:
_curTestSet.AppDomainLoaderMode = AppDomainLoaderMode.Lazy;
break;
default:
throw new Exception(String.Format("Unknown AD Loader mode {0} specified!", currentXML.Value));
}
break;
case configPercentPassIsPass:
_curTestSet.PercentPassIsPass = Convert.ToInt32(currentXML.Value);
break;
case configDefaultDebugger:
if (currentXML.Value.Length >= 7 && currentXML.Value.Substring(currentXML.Value.Length - 7).ToLower() == "cdb.exe")
{
_curTestSet.DefaultDebugger = currentXML.Value;
}
else if (currentXML.Value.Length >= 10 && currentXML.Value.Substring(currentXML.Value.Length - 7).ToLower() == "windbg.exe")
{
_curTestSet.DefaultDebugger = currentXML.Value;
}
else if (currentXML.Value.ToLower() == "none")
{
_curTestSet.DefaultDebugger = String.Empty;
}
else
{
throw new Exception("Unknown default debugger specified (" + currentXML.Value + ")");
}
break;
case configDefaultDebuggerOptions:
_curTestSet.DefaultDebuggerOptions = Environment.ExpandEnvironmentVariables(currentXML.Value);
break;
case configULAssemblyLoadPercent:
_curTestSet.ULAssemblyLoadPercent = Convert.ToInt32(currentXML.Value);
break;
case configULAppDomainUnloadPercent:
_curTestSet.ULAppDomainUnloadPercent = Convert.ToInt32(currentXML.Value);
break;
case configULGeneralUnloadPercent:
_curTestSet.ULGeneralUnloadPercent = Convert.ToInt32(currentXML.Value);
break;
case configULWaitTime:
_curTestSet.ULWaitTime = Convert.ToInt32(currentXML.Value);
break;
case configCcFailMail:
_curTestSet.CCFailMail = currentXML.Value;
break;
default:
throw new Exception("Unknown attribute (" + currentXML.Name + ") on " + concurrentConfigTest + " tag!");
}
}
// Check to see if any of the test attribute environment variables are set,
// If so, then use the environment variables.
if ((Environment.GetEnvironmentVariable("TIMELIMIT") != null) && (Environment.GetEnvironmentVariable("TIMELIMIT") != ""))
_curTestSet.MaximumTime = ConvertTimeValueToTestRunTime(Environment.GetEnvironmentVariable("TIMELIMIT"));
if ((Environment.GetEnvironmentVariable("MINCPU") != null) && (Environment.GetEnvironmentVariable("MINCPU") != ""))
_curTestSet.MinPercentCPU = Convert.ToInt32(Environment.GetEnvironmentVariable("MINCPU"));
_testSet.Add(_curTestSet);
break;
case configDiscovery:
if (testLevelStack.Count == 0 || (string)testLevelStack.Peek() != concurrentConfigTest)
{
throw new ArgumentException("The assembly tag can only appear as a child to the test tag (curent parent tag==" + (string)testLevelStack.Peek() + ").");
}
testLevelStack.Push(configDiscovery);
string path = null;
while (currentXML.MoveToNextAttribute())
{
XmlDebugOut(" " + currentXML.Name + "=\"" + currentXML.Value + "\"");
switch (currentXML.Name)
{
case configDiscoveryPath:
path = currentXML.Value;
break;
default:
throw new Exception("Unknown attribute on include tag (\"" + currentXML.Name + "\")!");
}
}
discoveryPaths.Add(Environment.ExpandEnvironmentVariables(path));
break;
case concurrentConfigAssembly:
/***********************************************************************
* Here's where we process an assembly & it's options. *
***********************************************************************/
bool disabled = false;
if (testLevelStack.Count == 0 || (string)testLevelStack.Peek() != concurrentConfigTest)
{
throw new ArgumentException("The assembly tag can only appear as a child to the test tag (curent parent tag==" + (string)testLevelStack.Peek() + ").");
}
testLevelStack.Push(concurrentConfigAssembly);
ReliabilityTest rt = new ReliabilityTest(_curTestSet.SuppressConsoleOutputFromTests);
rt.TestStartMode = _curTestSet.DefaultTestStartMode;
// first we need to setup any default options which are set globally on
// the test start mode.
if (null != _curTestSet.DefaultDebugger)
{
if (_curTestSet.DefaultTestStartMode != TestStartModeEnum.ProcessLoader)
{
throw new Exception(String.Format("{0} specified with default debugger or debugger options. If you want a debugger per test please use {1}=\"{2}\" ",
configTestStartModeAppDomainLoader,
configDefaultTestStartMode,
configTestStartModeProcessLoader));
}
rt.Debugger = _curTestSet.DefaultDebugger;
}
if (null != _curTestSet.DefaultDebuggerOptions)
{
if (_curTestSet.DefaultTestStartMode != TestStartModeEnum.ProcessLoader)
{
throw new Exception(String.Format("{0} specified with default debugger or debugger options. If you want a debugger per test please use {1}=\"{2}\" ",
configTestStartModeAppDomainLoader,
configDefaultTestStartMode,
configTestStartModeProcessLoader));
}
rt.DebuggerOptions = _curTestSet.DefaultDebuggerOptions;
}
// then we need to process the individual options & overrides.
while (currentXML.MoveToNextAttribute())
{
XmlDebugOut(" " + currentXML.Name + "=\"" + currentXML.Value + "\"");
switch (currentXML.Name)
{
case configAssemblyName:
rt.RefOrID = currentXML.Value;
break;
case configAssemblyBasePath:
rt.BasePath = Environment.ExpandEnvironmentVariables(currentXML.Value);
break;
case configAssemblyRequiresSDK:
if (String.Compare(currentXML.Value, "true", true) == 0 ||
currentXML.Value == "1" ||
String.Compare(currentXML.Value, "yes", true) == 0)
{
rt.RequiresSDK = true;
}
else if (String.Compare(currentXML.Value, "false", true) == 0 ||
currentXML.Value == "0" ||
String.Compare(currentXML.Value, "no", true) == 0)
{
rt.RequiresSDK = false;
}
else
{
throw new Exception("RequiresSDK has illegal value. Must be true, 1, yes, false, 0, or no");
}
break;
case configAssemblyFilename:
rt.Assembly = Environment.ExpandEnvironmentVariables(currentXML.Value);
Console.WriteLine("test is " + rt.Assembly);
break;
case configAssemblySuccessCode:
rt.SuccessCode = Convert.ToInt32(currentXML.Value);
break;
case configAssemblyEntryPoint:
rt.Arguments = currentXML.Value;
break;
case configAssemblyArguments:
if (!string.IsNullOrEmpty(currentXML.Value))
rt.Arguments = Environment.ExpandEnvironmentVariables(currentXML.Value);
break;
case configAssemblyConcurrentCopies:
rt.ConcurrentCopies = Convert.ToInt32(currentXML.Value);
break;
case configAssemblyStatus:
if (currentXML.Value == configAssemblyStatusDisabled)
{
disabled = true;
}
break;
case configAssemblyDebugger:
if (TestStartModeEnum.ProcessLoader != _curTestSet.DefaultTestStartMode)
{
throw new Exception(String.Format("{0} can only be set for test sets with {1}=\"{2}\" set.",
configAssemblyDebugger,
configDefaultTestStartMode,
configTestStartModeProcessLoader));
}
if (currentXML.Value.Length >= 7 && currentXML.Value.Substring(currentXML.Value.Length - 7).ToLower() == "cdb.exe")
{
rt.Debugger = currentXML.Value;
}
else if (currentXML.Value.Length >= 10 && currentXML.Value.Substring(currentXML.Value.Length - 7).ToLower() == "windbg.exe")
{
rt.Debugger = currentXML.Value;
}
else if (currentXML.Value.ToLower() == "none")
{
rt.Debugger = String.Empty;
}
else
{
throw new Exception("Unknown debugger specified (" + currentXML.Value + ")");
}
break;
case configAssemblyDebuggerOptions:
if (TestStartModeEnum.ProcessLoader != _curTestSet.DefaultTestStartMode)
{
throw new Exception(String.Format("{0} can only be set for test sets with {1}=\"{2}\" set.",
configAssemblyDebuggerOptions,
configDefaultTestStartMode,
configTestStartModeProcessLoader));
}
rt.DebuggerOptions = Environment.ExpandEnvironmentVariables(currentXML.Value);
break;
case configAssemblySmartNetGuid:
try
{
rt.Guid = new Guid(currentXML.Value);
}
catch (FormatException)
{
throw new Exception(String.Format("The format for guid {0} on test {1} is invalid", currentXML.Value, rt.RefOrID));
}
break;
case configAssemblyDuration:
if (currentXML.Value.IndexOf(":") == -1)
{
// just a number of minutes
rt.ExpectedDuration = Convert.ToInt32(currentXML.Value);
}
else
{
// time span
try
{
rt.ExpectedDuration = unchecked((int)(TimeSpan.Parse(currentXML.Value).Ticks / TimeSpan.TicksPerMinute));
}
catch
{
throw new Exception(String.Format("Bad time span {0} for expected duration.", currentXML.Value));
}
}
break;
case configAssemblyTestAttributes:
string[] attrs = currentXML.Value.Split(';');
TestAttributes testAttrs = TestAttributes.None;
for (int j = 0; j < attrs.Length; j++)
{
switch (attrs[j].ToLower())
{
case "requiressta":
testAttrs |= TestAttributes.RequiresSTAThread;
break;
case "requiresmta":
testAttrs |= TestAttributes.RequiresMTAThread;
break;
default:
throw new Exception(String.Format("Unknown test attribute: {0}", attrs[j]));
}
}
rt.TestAttrs = testAttrs;
break;
case configAssemblyTestLoader:
switch (currentXML.Value)
{
case configTestStartModeAppDomainLoader:
if (null != rt.Debugger || null != rt.DebuggerOptions)
{
throw new Exception(String.Format("{0} specified with debugger or debugger options. If you want a debugger per test please use {1}=\"{2}\" ",
configTestStartModeAppDomainLoader,
configDefaultTestStartMode,
configTestStartModeProcessLoader));
}
rt.TestStartMode = TestStartModeEnum.AppDomainLoader;
break;
case configTestStartModeProcessLoader:
rt.TestStartMode = TestStartModeEnum.ProcessLoader;
break;
default:
throw new Exception(String.Format("Unknown test starter {0} specified!", currentXML.Value));
}
break;
case configAssemblyTestOwner:
rt.TestOwner = currentXML.Value;
break;
case configAssemblyTestGroup:
string groupName = currentXML.Value;
// first, we want to see if another test has this group. We store the group name in
// our group List as the 1st entry. If we find a group we set our List
// arraylist to that same List (and add ourselves to it). We're then all in
// one group, the arraylist.
int i = 0;
for (i = 0; i < foundTests.Count; i++)
{
ReliabilityTest test = foundTests[i] as ReliabilityTest;
Debug.Assert(test != null, "Non reliability test in foundTests array!");
if (null != test.Group)
{
string curGroupName = test.Group[0].ToString();
if (String.Compare(curGroupName, groupName, false) == 0)
{
test.Group.Add(rt);
rt.Group = test.Group;
break;
}
}
}
if (rt.Group == null)
{
// this is the first test in this group
rt.Group = new List<ReliabilityTest>();
rt.Group.Add(rt);
}
break;
case configAssemblyPostCommand:
if (rt.PostCommands == null)
{
// first pre command on this test
rt.PostCommands = new List<string>();
}
rt.PostCommands.Add(Environment.ExpandEnvironmentVariables(currentXML.Value));
break;
case configAssemblyPreCommand:
if (rt.PreCommands == null)
{
// first pre command on this test
rt.PreCommands = new List<string>();
}
rt.PreCommands.Add(Environment.ExpandEnvironmentVariables(currentXML.Value));
break;
case configAssemblyCustomAction:
switch (currentXML.Value)
{
case "LegacySecurityPolicy":
rt.CustomAction = CustomActionType.LegacySecurityPolicy;
break;
default:
throw new Exception(String.Format("Unknown custom action: {0}", currentXML.Value));
}
break;
default:
throw new Exception("Unknown attribute on assembly tag (" + currentXML.Name + "=" + currentXML.Value + ")");
}
}
// if the test is disabled or it requires the SDK to be installed &
// we don't have the SDK installed then don't add it to our list
// of tests to run.
if (disabled || (rt.RequiresSDK == true && Environment.GetEnvironmentVariable("INSTALL_SDK") == null))
{
break;
}
int testCopies = 1;
if (_curTestSet.AppDomainLoaderMode == AppDomainLoaderMode.FullIsolation)
{
// in this mode each copy of the test is ran in it's own app domain,
// fully isolated from all other copies of the test. If the user
// specified a cloning level we need to duplicate the test.
testCopies = rt.ConcurrentCopies;
rt.ConcurrentCopies = 1;
}
else if (_curTestSet.AppDomainLoaderMode == AppDomainLoaderMode.RoundRobin)
{
// In this mode each test is ran in an app domain w/ other tests.
testCopies = rt.ConcurrentCopies;
rt.ConcurrentCopies = 1;
}
else
{
// Normal mode - tests are ran in app domains w/ copies of themselves
}
string refOrId = rt.RefOrID;
if (rt.RefOrID == null || rt.RefOrID == String.Empty)
{
refOrId = rt.Assembly + rt.Arguments;
}
for (int j = 0; j < testCopies; j++)
{
if (testCopies > 1)
{
rt.RefOrID = String.Format("{0} Copy {1}", refOrId, j);
}
else
{
rt.RefOrID = refOrId;
}
bool fRetry;
do
{
fRetry = false;
for (int i = 0; i < foundTests.Count; i++)
{
if (((ReliabilityTest)foundTests[i]).RefOrID == rt.RefOrID)
{
rt.RefOrID = rt.RefOrID + "_" + i.ToString();
fRetry = true;
break;
}
}
} while (fRetry);
ReliabilityTest clone = (ReliabilityTest)rt.Clone();
clone.Index = foundTests.Add(clone);
}
break;
default:
throw new ArgumentException("Unknown node (\"" + currentXML.NodeType + "\") named \"" + currentXML.Name + "\"=\"" + currentXML.Value + "\" in config file!");
} // end of switch(currentXML.Name)
if (isEmpty)
{
XmlDebugOut("/>\r\n");
testLevelStack.Pop();
}
else
{
XmlDebugOut(">\r\n");
}
break;
case XmlNodeType.Text:
case XmlNodeType.CDATA:
case XmlNodeType.ProcessingInstruction:
case XmlNodeType.Comment:
case XmlNodeType.Document:
case XmlNodeType.Whitespace:
case XmlNodeType.SignificantWhitespace:
break;
case XmlNodeType.EndElement:
IndentToDepth(totalDepth + currentXML.Depth);
XmlDebugOutLine("</" + currentXML.Name + ">");
// note: we never pop or push the includes tag. It's a special 'hidden' tag
// we should also never have to pop a configInclude tag, but it might happen
if (currentXML.Name != configIncludes && currentXML.Name != configInclude && currentXML.Name != configHost)
{
testLevelStack.Pop();
}
break;
} // end of switch(currentXML.NodeType)
} // end of while(currentXML.Read())
} while (xmlFileStack.Count > 0);
if (_curTestSet != null && foundTests != null && foundTests.Count > 0)
{
_curTestSet.Tests = (ReliabilityTest[])foundTests.ToArray(typeof(ReliabilityTest));
_curTestSet.DiscoveryPaths = (string[])discoveryPaths.ToArray(typeof(string));
discoveryPaths.Clear();
foundTests.Clear();
}
}
