/// <summary>
/// Tries to unload a random appdomain every ULWaitTime milliseconds while tests are running
/// </summary>
public void GeneralUnload()
{
while (true)
{
Thread.Sleep(curTestSet.ULWaitTime);
if (randNum.Next(100) < curTestSet.ULGeneralUnloadPercent)
{
ArrayList availableDomains = new ArrayList();
//figure out which domains are still available
for (int i = 0; i < curTestSet.Tests.Length; i++)
{
if (curTestSet.Tests[i].AppDomain != null)
{
availableDomains.Add(curTestSet.Tests[i]);
}
}
//if we found at least one appDomain, pick one to unload
if (availableDomains.Count > 0)
{
int x = randNum.Next(availableDomains.Count - 1);
lock ((ReliabilityTest)availableDomains[x])
{
if (((ReliabilityTest)availableDomains[x]).AppDomain != null)
{
Console.WriteLine("ReliabilityFramework: {0}-{1}-{2}", eReasonForUnload.GeneralUnload, ((ReliabilityTest)availableDomains[x]).RefOrID, System.DateTime.Now);
//print unload message to log
///WriteToLog(String.Format("UNLOAD {0} AppDomain Unloaded: {1}{2}", DateTime.Now, ((ReliabilityTest)availableDomains[x]).RefOrID, Environment.NewLine));
//update testsRanCount to reflect missing tests
testsRanCount += ((ReliabilityTest)availableDomains[x]).ConcurrentCopies * curTestSet.MaximumLoops - ((ReliabilityTest)availableDomains[x]).RunCount;
// unload domain
AppDomainUnloadDelegate adu = new AppDomainUnloadDelegate(AppDomain.Unload);
adu.BeginInvoke(((ReliabilityTest)availableDomains[x]).AppDomain, null, null);
((ReliabilityTest)availableDomains[x]).AppDomain = null;
((ReliabilityTest)availableDomains[x]).ConcurrentCopies = 0;
}
}
}
else
{
return;
}
}
}
}
/// <summary>
/// Handles unloads for assemblyLoad and domainUnload events
/// </summary>
/// <param name="ad">The appdomain to potentially unload</param>
/// <param name="eReasonForUnload">reason for unload</param>
public void UnloadOnEvent(AppDomain ad, eReasonForUnload reason)
{
int percent = 0;
switch (reason)
{
case eReasonForUnload.AssemblyLoad:
percent = curTestSet.ULAssemblyLoadPercent;
break;
case eReasonForUnload.AppDomainUnload:
percent = curTestSet.ULAppDomainUnloadPercent;
break;
case eReasonForUnload.Replay:
percent = 100;
break;
}
if (randNum.Next(100) < percent)
{
for (int i = 0; i < curTestSet.Tests.Length; i++)
{
lock (curTestSet.Tests[i])
{
if ((curTestSet.Tests[i].AppDomain != null) && (curTestSet.Tests[i].AppDomain.FriendlyName == ad.FriendlyName))
{
//update testsRanCount to reflect missing tests
this.testsRanCount += curTestSet.Tests[i].ConcurrentCopies * curTestSet.MaximumLoops - curTestSet.Tests[i].RunCount;
//print unload message to log
//WriteToLog(String.Format("UNLOAD {0} AppDomain Unloaded: {1}{2}", DateTime.Now, curTestSet.Tests[i].RefOrID, Environment.NewLine));
if (reason == eReasonForUnload.AssemblyLoad)
{
//The test isn't going to load, so don't count it
Interlocked.Decrement(ref LoadingCount);
}
Console.WriteLine("\nReliabilityFramework: Unload on{0}-{1}-{2}", reason.ToString(), curTestSet.Tests[i].RefOrID, System.DateTime.Now);
// unload domain
AppDomainUnloadDelegate adu = new AppDomainUnloadDelegate(AppDomain.Unload);
adu.BeginInvoke(curTestSet.Tests[i].AppDomain, null, null);
curTestSet.Tests[i].AppDomain = null;
curTestSet.Tests[i].ConcurrentCopies = 0;
}
}
}
}
}
/// <summary>
/// StartTestWorker does the actual work of starting a test. It should already be running on it's own thread by the time
/// we call here (because StartTest creates the new thread).
/// </summary>
private void StartTestWorker(object test)
{
try
{
// Update the running time for the stress run
if (((TimeSpan)DateTime.Now.Subtract(_lastLogTime)).TotalMinutes >= 30)
{
_logger.RecordTimeStamp();
_lastLogTime = DateTime.Now;
}
ReliabilityTest daTest = test as ReliabilityTest;
if (_detourHelpers != null)
{
_detourHelpers.SetTestIdName(daTest.Index + 1, daTest.RefOrID);
_detourHelpers.SetThreadTestId(daTest.Index + 1);
}
Debug.Assert(daTest != null); // if we didn't find the test then there's something horribly wrong!
daTest.StartTime = DateTime.Now;
switch (daTest.TestStartMode)
{
case TestStartModeEnum.ProcessLoader:
#if PROJECTK_BUILD
Task.Factory.StartNew(() =>
{
Console.WriteLine("==============================running test: {0}==============================", daTest.Assembly);
try
{
daTest.EntryPointMethod.Invoke(null, new object[] { (daTest.Arguments == null) ? new string[0] : daTest.GetSplitArguments() });
}
catch (Exception e)
{
Console.WriteLine(e);
}
Interlocked.Increment(ref _testsRanCount);
SignalTestFinished(daTest);
});
#else
try
{
if (daTest.TestObject is string)
{
ProcessStartInfo pi = null;
Process p = null;
try
{
if (daTest.Debugger != null && daTest.Debugger != String.Empty)
{
if (daTest.DebuggerOptions == null)
{
pi = new ProcessStartInfo((string)daTest.Debugger,
String.Format("{0} {1} {2}",
"-g",
(string)daTest.TestObject,
daTest.Arguments
));
}
else
{
pi = new ProcessStartInfo((string)daTest.Debugger,
String.Format("{0} {1} {2}",
daTest.DebuggerOptions,
(string)daTest.TestObject,
daTest.Arguments
));
}
}
else
{
// works for both exe and batch files
// a temporary file for exitcode is not needed here
// since it's not running directly under smarty
pi = new ProcessStartInfo((string)daTest.TestObject, daTest.Arguments);
}
if (daTest.BasePath != String.Empty)
{
pi.WorkingDirectory = daTest.BasePath;
}
pi.UseShellExecute = false;
pi.RedirectStandardOutput = true;
pi.RedirectStandardError = true;
p = Process.Start(pi);
using (ProcessReader pr = new ProcessReader(p, daTest.RefOrID)) // reads & outputs standard error & output
{
int timeOutCnt = 0;
while (!pr.Exited)
{
if (timeOutCnt > 24 * 15)
{
pr.TimedOut = true;
break;
}
Thread.Sleep(2500);
timeOutCnt++;
}
int exitCode = 0;
if (!pr.TimedOut)
{
p.WaitForExit();
exitCode = p.ExitCode;
if (exitCode != daTest.SuccessCode)
{
AddFailure(String.Format("Test Result ({0}) != Success ({1})", exitCode, daTest.SuccessCode), daTest, exitCode);
}
else
{
AddSuccess("", daTest, exitCode);
}
}
else
{
// external test timed out..
try
{
p.Kill();
AddFailure(String.Format("Test Timed out after 15 minutes..."), daTest, exitCode);
}
catch (Exception e)
{
AddFailure(String.Format("Test timed out after 15 minutes, failed to kill: {0}", e), daTest, exitCode);
}
}
}
}
catch (Exception e)
{
if (null != pi)
{
// let's see what process info thinks of this...
Console.WriteLine("Trying to run {1} {2} from {0}", pi.WorkingDirectory, pi.FileName, pi.Arguments);
}
Console.WriteLine("Unable to start test: {0} because {1} was thrown ({2})\r\nStack Trace: {3}", daTest.TestObject, e.GetType(), e.Message, e.StackTrace);
SignalTestFinished(daTest);
AddFailure("Unable to run test: ", daTest, -1);
return;
}
finally
{
if (null != p)
{
p.Dispose();
}
}
Interlocked.Increment(ref testsRanCount);
SignalTestFinished(daTest);
}
else if (daTest.TestObject != null)
{
SignalTestFinished(daTest);
throw new NotSupportedException("ProcessLoader can only load applications, not assemblies!" + daTest.RefOrID + daTest.TestObject.GetType().ToString());
}
else
{
SignalTestFinished(daTest);
AddFailure("Test does not exist", daTest, -1);
}
}
catch (Exception e)
{
AddFailure("Failed to start test in process loader mode" + e.ToString(), daTest, -1);
Interlocked.Increment(ref testsRanCount);
SignalTestFinished(daTest);
Console.WriteLine("STA thread issue encountered, couldn't start in process loader {0}", e);
MyDebugBreak(daTest.RefOrID);
}
#endif
break;
case TestStartModeEnum.AppDomainLoader:
#if PROJECTK_BUILD
Console.WriteLine("Appdomain mode is NOT supported for ProjectK");
#else
try
{
if (daTest.TestObject is string)
{
try
{
int exitCode;
#pragma warning disable 0618
// AppendPrivatePath is obsolute, should move to AppDomainSetup but it's too risky at end of beta 1.
daTest.AppDomain.AppendPrivatePath(daTest.BasePath);
#pragma warning restore
// Execute the test.
if (daTest.Assembly.ToLower().IndexOf(".exe") == -1 && daTest.Assembly.ToLower().IndexOf(".dll") == -1) // must be a simple name or fullname...
{
//exitCode = daTest.AppDomain.ExecuteAssemblyByName(daTest.Assembly, daTest.GetSplitArguments());
exitCode = daTest.AppDomain.ExecuteAssemblyByName(daTest.Assembly, null, daTest.GetSplitArguments());
}
else
{
//exitCode = daTest.AppDomain.ExecuteAssembly(daTest.Assembly, daTest.GetSplitArguments());
exitCode = daTest.AppDomain.ExecuteAssembly(daTest.Assembly, null, daTest.GetSplitArguments());
}
// HACKHACK: VSWhidbey bug #113535: Breaking change. Tests that return a value via Environment.ExitCode
// will not have their value propagated back properly via AppDomain.ExecuteAssembly. These tests will
// typically have a return value of 0 (because they have a void entry point). We will check
// Environment.ExitCode and if it's not zero, we'll treat that as our return value (then reset
// Env.ExitCode back to 0).
if (exitCode == 0 && Environment.ExitCode != 0)
{
logger.WriteTestRace(daTest);
exitCode = Environment.ExitCode;
Environment.ExitCode = 0;
}
if (exitCode != daTest.SuccessCode)
{
AddFailure(String.Format("Test Result ({0}) != Success ({1})", exitCode, daTest.SuccessCode), daTest, exitCode);
}
else
{
AddSuccess("", daTest, exitCode);
}
logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.Tests, String.Format("Test {0} has exited with result {1}", daTest.RefOrID, exitCode));
}
catch (PathTooLongException)
{
if (curTestSet.DebugBreakOnPathTooLong)
{
MyDebugBreak("Path too long");
}
}
catch (AppDomainUnloadedException)
{
logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.Tests, String.Format("Error in executing test {0}: AppDomainUnloaded", daTest.RefOrID));
AddFailure("Failed to ExecuteAssembly, AD Unloaded (" + daTest.AppDomain.FriendlyName + ")", daTest, -1);
}
catch (OutOfMemoryException e)
{
HandleOom(e, "ExecuteAssembly");
}
catch (Exception e)
{
Exception eTemp = e.InnerException;
while (eTemp != null)
{
if (eTemp is OutOfMemoryException)
{
HandleOom(e, "ExecuteAssembly (inner)");
break;
}
eTemp = eTemp.InnerException;
}
if (eTemp == null)
{
logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.Tests, String.Format("Error in executing test {0}: {1}", daTest.RefOrID, e));
AddFailure("Failed to ExecuteAssembly (" + e.ToString() + ")", daTest, -1);
}
if ((Thread.CurrentThread.ThreadState & System.Threading.ThreadState.AbortRequested) != 0)
{
UnexpectedThreadAbortDebugBreak();
logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.Tests, String.Format("Test left thread w/ abort requested set"));
AddFailure("Abort Requested Bit Still Set (" + e.ToString() + ")", daTest, -1);
Thread.ResetAbort();
}
}
}
else if (daTest.TestObject is ISingleReliabilityTest)
{
try
{
if (((ISingleReliabilityTest)daTest.TestObject).Run() != true)
{
AddFailure("SingleReliabilityTest returned false", daTest, 0);
}
else
{
AddSuccess("", daTest, 1);
}
}
catch (Exception e)
{
Console.WriteLine("Error in executing ISingleReliabilityTest: {0}", e);
AddFailure("ISingleReliabilityTest threw exception!", daTest, -1);
}
}
else if (daTest.TestObject is IMultipleReliabilityTest)
{
try
{
if (((IMultipleReliabilityTest)daTest.TestObject).Run(0) != true)
{
AddFailure("MultipleReliabilityTest returned false", daTest, 0);
}
else
{
AddSuccess("", daTest, 1);
}
}
catch (Exception ex)
{
Console.WriteLine("Error in executing IMultipleReliabilityTest: {0}", ex);
AddFailure("IMultipleReliabilityTest threw exception!", daTest, -1);
}
}
/* Test is finished executing, we need to clean up now... */
Interlocked.Increment(ref testsRanCount);
if (curTestSet.AppDomainLoaderMode == AppDomainLoaderMode.FullIsolation || curTestSet.AppDomainLoaderMode == AppDomainLoaderMode.Lazy)
{
// we're in full isolation & have test runs left. we need to
// recreate the app domain so that we don't die on statics.
lock (daTest)
{
if (daTest.AppDomain != null)
{
logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.AppDomain, String.Format("Unloading app domain (locked): {0}", daTest.AppDomain.FriendlyName));
AppDomain.Unload(daTest.AppDomain);
daTest.AppDomain = null;
daTest.MyLoader = null;
}
if (curTestSet.MaximumLoops != 1 && curTestSet.AppDomainLoaderMode != AppDomainLoaderMode.Lazy)
{
TestPreLoader(daTest, curTestSet.DiscoveryPaths); // need to reload assembly & appdomain
}
logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.AppDomain, String.Format("Unloading complete (freeing lock): {0}", daTest.RefOrID));
}
}
else if ((daTest.RunCount >= (curTestSet.MaximumLoops * daTest.ConcurrentCopies) ||
((DateTime.Now.Subtract(startTime).Ticks / TimeSpan.TicksPerMinute > curTestSet.MaximumTime))) &&
(curTestSet.AppDomainLoaderMode != AppDomainLoaderMode.RoundRobin)) // don't want to unload domains in round robin mode, we don't know how
// many tests are left.
{
lock (daTest)
{ // make sure no one accesses the app domain at the same time (between here & RunReliabilityTests)
if (daTest.RunningCount == 1 && daTest.AppDomain != null)
{ // only unload when the last test finishes.
// unload app domain's async so that we don't block while holding daTest lock.
AppDomainUnloadDelegate adu = new AppDomainUnloadDelegate(AppDomain.Unload);
adu.BeginInvoke(daTest.AppDomain, null, null);
daTest.AppDomain = null;
RunCommands(daTest.PostCommands, "post", daTest);
}
}
}
}
catch (ThreadAbortException)
{
UnexpectedThreadAbortDebugBreak();
}
finally
{
SignalTestFinished(daTest);
}
#endif
break;
}
}
catch (Exception e)
{
_logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, String.Format("Unexpected exception on StartTestWorker: {0}", e));
}
}
