/// <summary> /// Tries to unload a random appdomain every ULWaitTime milliseconds while tests are running /// </summary> public void GeneralUnload() { while (true) { Thread.Sleep(curTestSet.ULWaitTime); if (randNum.Next(100) < curTestSet.ULGeneralUnloadPercent) { ArrayList availableDomains = new ArrayList(); //figure out which domains are still available for (int i = 0; i < curTestSet.Tests.Length; i++) { if (curTestSet.Tests[i].AppDomain != null) { availableDomains.Add(curTestSet.Tests[i]); } } //if we found at least one appDomain, pick one to unload if (availableDomains.Count > 0) { int x = randNum.Next(availableDomains.Count - 1); lock ((ReliabilityTest)availableDomains[x]) { if (((ReliabilityTest)availableDomains[x]).AppDomain != null) { Console.WriteLine("ReliabilityFramework: {0}-{1}-{2}", eReasonForUnload.GeneralUnload, ((ReliabilityTest)availableDomains[x]).RefOrID, System.DateTime.Now); //print unload message to log ///WriteToLog(String.Format("UNLOAD {0} AppDomain Unloaded: {1}{2}", DateTime.Now, ((ReliabilityTest)availableDomains[x]).RefOrID, Environment.NewLine)); //update testsRanCount to reflect missing tests testsRanCount += ((ReliabilityTest)availableDomains[x]).ConcurrentCopies * curTestSet.MaximumLoops - ((ReliabilityTest)availableDomains[x]).RunCount; // unload domain AppDomainUnloadDelegate adu = new AppDomainUnloadDelegate(AppDomain.Unload); adu.BeginInvoke(((ReliabilityTest)availableDomains[x]).AppDomain, null, null); ((ReliabilityTest)availableDomains[x]).AppDomain = null; ((ReliabilityTest)availableDomains[x]).ConcurrentCopies = 0; } } } else { return; } } } }
/// <summary> /// Handles unloads for assemblyLoad and domainUnload events /// </summary> /// <param name="ad">The appdomain to potentially unload</param> /// <param name="eReasonForUnload">reason for unload</param> public void UnloadOnEvent(AppDomain ad, eReasonForUnload reason) { int percent = 0; switch (reason) { case eReasonForUnload.AssemblyLoad: percent = curTestSet.ULAssemblyLoadPercent; break; case eReasonForUnload.AppDomainUnload: percent = curTestSet.ULAppDomainUnloadPercent; break; case eReasonForUnload.Replay: percent = 100; break; } if (randNum.Next(100) < percent) { for (int i = 0; i < curTestSet.Tests.Length; i++) { lock (curTestSet.Tests[i]) { if ((curTestSet.Tests[i].AppDomain != null) && (curTestSet.Tests[i].AppDomain.FriendlyName == ad.FriendlyName)) { //update testsRanCount to reflect missing tests this.testsRanCount += curTestSet.Tests[i].ConcurrentCopies * curTestSet.MaximumLoops - curTestSet.Tests[i].RunCount; //print unload message to log //WriteToLog(String.Format("UNLOAD {0} AppDomain Unloaded: {1}{2}", DateTime.Now, curTestSet.Tests[i].RefOrID, Environment.NewLine)); if (reason == eReasonForUnload.AssemblyLoad) { //The test isn't going to load, so don't count it Interlocked.Decrement(ref LoadingCount); } Console.WriteLine("\nReliabilityFramework: Unload on{0}-{1}-{2}", reason.ToString(), curTestSet.Tests[i].RefOrID, System.DateTime.Now); // unload domain AppDomainUnloadDelegate adu = new AppDomainUnloadDelegate(AppDomain.Unload); adu.BeginInvoke(curTestSet.Tests[i].AppDomain, null, null); curTestSet.Tests[i].AppDomain = null; curTestSet.Tests[i].ConcurrentCopies = 0; } } } } }
/// <summary> /// StartTestWorker does the actual work of starting a test. It should already be running on it's own thread by the time /// we call here (because StartTest creates the new thread). /// </summary> private void StartTestWorker(object test) { try { // Update the running time for the stress run if (((TimeSpan)DateTime.Now.Subtract(_lastLogTime)).TotalMinutes >= 30) { _logger.RecordTimeStamp(); _lastLogTime = DateTime.Now; } ReliabilityTest daTest = test as ReliabilityTest; if (_detourHelpers != null) { _detourHelpers.SetTestIdName(daTest.Index + 1, daTest.RefOrID); _detourHelpers.SetThreadTestId(daTest.Index + 1); } Debug.Assert(daTest != null); // if we didn't find the test then there's something horribly wrong! daTest.StartTime = DateTime.Now; switch (daTest.TestStartMode) { case TestStartModeEnum.ProcessLoader: #if PROJECTK_BUILD Task.Factory.StartNew(() => { Console.WriteLine("==============================running test: {0}==============================", daTest.Assembly); try { daTest.EntryPointMethod.Invoke(null, new object[] { (daTest.Arguments == null) ? new string[0] : daTest.GetSplitArguments() }); } catch (Exception e) { Console.WriteLine(e); } Interlocked.Increment(ref _testsRanCount); SignalTestFinished(daTest); }); #else try { if (daTest.TestObject is string) { ProcessStartInfo pi = null; Process p = null; try { if (daTest.Debugger != null && daTest.Debugger != String.Empty) { if (daTest.DebuggerOptions == null) { pi = new ProcessStartInfo((string)daTest.Debugger, String.Format("{0} {1} {2}", "-g", (string)daTest.TestObject, daTest.Arguments )); } else { pi = new ProcessStartInfo((string)daTest.Debugger, String.Format("{0} {1} {2}", daTest.DebuggerOptions, (string)daTest.TestObject, daTest.Arguments )); } } else { // works for both exe and batch files // a temporary file for exitcode is not needed here // since it's not running directly under smarty pi = new ProcessStartInfo((string)daTest.TestObject, daTest.Arguments); } if (daTest.BasePath != String.Empty) { pi.WorkingDirectory = daTest.BasePath; } pi.UseShellExecute = false; pi.RedirectStandardOutput = true; pi.RedirectStandardError = true; p = Process.Start(pi); using (ProcessReader pr = new ProcessReader(p, daTest.RefOrID)) // reads & outputs standard error & output { int timeOutCnt = 0; while (!pr.Exited) { if (timeOutCnt > 24 * 15) { pr.TimedOut = true; break; } Thread.Sleep(2500); timeOutCnt++; } int exitCode = 0; if (!pr.TimedOut) { p.WaitForExit(); exitCode = p.ExitCode; if (exitCode != daTest.SuccessCode) { AddFailure(String.Format("Test Result ({0}) != Success ({1})", exitCode, daTest.SuccessCode), daTest, exitCode); } else { AddSuccess("", daTest, exitCode); } } else { // external test timed out.. try { p.Kill(); AddFailure(String.Format("Test Timed out after 15 minutes..."), daTest, exitCode); } catch (Exception e) { AddFailure(String.Format("Test timed out after 15 minutes, failed to kill: {0}", e), daTest, exitCode); } } } } catch (Exception e) { if (null != pi) { // let's see what process info thinks of this... Console.WriteLine("Trying to run {1} {2} from {0}", pi.WorkingDirectory, pi.FileName, pi.Arguments); } Console.WriteLine("Unable to start test: {0} because {1} was thrown ({2})\r\nStack Trace: {3}", daTest.TestObject, e.GetType(), e.Message, e.StackTrace); SignalTestFinished(daTest); AddFailure("Unable to run test: ", daTest, -1); return; } finally { if (null != p) { p.Dispose(); } } Interlocked.Increment(ref testsRanCount); SignalTestFinished(daTest); } else if (daTest.TestObject != null) { SignalTestFinished(daTest); throw new NotSupportedException("ProcessLoader can only load applications, not assemblies!" + daTest.RefOrID + daTest.TestObject.GetType().ToString()); } else { SignalTestFinished(daTest); AddFailure("Test does not exist", daTest, -1); } } catch (Exception e) { AddFailure("Failed to start test in process loader mode" + e.ToString(), daTest, -1); Interlocked.Increment(ref testsRanCount); SignalTestFinished(daTest); Console.WriteLine("STA thread issue encountered, couldn't start in process loader {0}", e); MyDebugBreak(daTest.RefOrID); } #endif break; case TestStartModeEnum.AppDomainLoader: #if PROJECTK_BUILD Console.WriteLine("Appdomain mode is NOT supported for ProjectK"); #else try { if (daTest.TestObject is string) { try { int exitCode; #pragma warning disable 0618 // AppendPrivatePath is obsolute, should move to AppDomainSetup but it's too risky at end of beta 1. daTest.AppDomain.AppendPrivatePath(daTest.BasePath); #pragma warning restore // Execute the test. if (daTest.Assembly.ToLower().IndexOf(".exe") == -1 && daTest.Assembly.ToLower().IndexOf(".dll") == -1) // must be a simple name or fullname... { //exitCode = daTest.AppDomain.ExecuteAssemblyByName(daTest.Assembly, daTest.GetSplitArguments()); exitCode = daTest.AppDomain.ExecuteAssemblyByName(daTest.Assembly, null, daTest.GetSplitArguments()); } else { //exitCode = daTest.AppDomain.ExecuteAssembly(daTest.Assembly, daTest.GetSplitArguments()); exitCode = daTest.AppDomain.ExecuteAssembly(daTest.Assembly, null, daTest.GetSplitArguments()); } // HACKHACK: VSWhidbey bug #113535: Breaking change. Tests that return a value via Environment.ExitCode // will not have their value propagated back properly via AppDomain.ExecuteAssembly. These tests will // typically have a return value of 0 (because they have a void entry point). We will check // Environment.ExitCode and if it's not zero, we'll treat that as our return value (then reset // Env.ExitCode back to 0). if (exitCode == 0 && Environment.ExitCode != 0) { logger.WriteTestRace(daTest); exitCode = Environment.ExitCode; Environment.ExitCode = 0; } if (exitCode != daTest.SuccessCode) { AddFailure(String.Format("Test Result ({0}) != Success ({1})", exitCode, daTest.SuccessCode), daTest, exitCode); } else { AddSuccess("", daTest, exitCode); } logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.Tests, String.Format("Test {0} has exited with result {1}", daTest.RefOrID, exitCode)); } catch (PathTooLongException) { if (curTestSet.DebugBreakOnPathTooLong) { MyDebugBreak("Path too long"); } } catch (AppDomainUnloadedException) { logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.Tests, String.Format("Error in executing test {0}: AppDomainUnloaded", daTest.RefOrID)); AddFailure("Failed to ExecuteAssembly, AD Unloaded (" + daTest.AppDomain.FriendlyName + ")", daTest, -1); } catch (OutOfMemoryException e) { HandleOom(e, "ExecuteAssembly"); } catch (Exception e) { Exception eTemp = e.InnerException; while (eTemp != null) { if (eTemp is OutOfMemoryException) { HandleOom(e, "ExecuteAssembly (inner)"); break; } eTemp = eTemp.InnerException; } if (eTemp == null) { logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.Tests, String.Format("Error in executing test {0}: {1}", daTest.RefOrID, e)); AddFailure("Failed to ExecuteAssembly (" + e.ToString() + ")", daTest, -1); } if ((Thread.CurrentThread.ThreadState & System.Threading.ThreadState.AbortRequested) != 0) { UnexpectedThreadAbortDebugBreak(); logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.Tests, String.Format("Test left thread w/ abort requested set")); AddFailure("Abort Requested Bit Still Set (" + e.ToString() + ")", daTest, -1); Thread.ResetAbort(); } } } else if (daTest.TestObject is ISingleReliabilityTest) { try { if (((ISingleReliabilityTest)daTest.TestObject).Run() != true) { AddFailure("SingleReliabilityTest returned false", daTest, 0); } else { AddSuccess("", daTest, 1); } } catch (Exception e) { Console.WriteLine("Error in executing ISingleReliabilityTest: {0}", e); AddFailure("ISingleReliabilityTest threw exception!", daTest, -1); } } else if (daTest.TestObject is IMultipleReliabilityTest) { try { if (((IMultipleReliabilityTest)daTest.TestObject).Run(0) != true) { AddFailure("MultipleReliabilityTest returned false", daTest, 0); } else { AddSuccess("", daTest, 1); } } catch (Exception ex) { Console.WriteLine("Error in executing IMultipleReliabilityTest: {0}", ex); AddFailure("IMultipleReliabilityTest threw exception!", daTest, -1); } } /* Test is finished executing, we need to clean up now... */ Interlocked.Increment(ref testsRanCount); if (curTestSet.AppDomainLoaderMode == AppDomainLoaderMode.FullIsolation || curTestSet.AppDomainLoaderMode == AppDomainLoaderMode.Lazy) { // we're in full isolation & have test runs left. we need to // recreate the app domain so that we don't die on statics. lock (daTest) { if (daTest.AppDomain != null) { logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.AppDomain, String.Format("Unloading app domain (locked): {0}", daTest.AppDomain.FriendlyName)); AppDomain.Unload(daTest.AppDomain); daTest.AppDomain = null; daTest.MyLoader = null; } if (curTestSet.MaximumLoops != 1 && curTestSet.AppDomainLoaderMode != AppDomainLoaderMode.Lazy) { TestPreLoader(daTest, curTestSet.DiscoveryPaths); // need to reload assembly & appdomain } logger.WriteToInstrumentationLog(curTestSet, LoggingLevels.AppDomain, String.Format("Unloading complete (freeing lock): {0}", daTest.RefOrID)); } } else if ((daTest.RunCount >= (curTestSet.MaximumLoops * daTest.ConcurrentCopies) || ((DateTime.Now.Subtract(startTime).Ticks / TimeSpan.TicksPerMinute > curTestSet.MaximumTime))) && (curTestSet.AppDomainLoaderMode != AppDomainLoaderMode.RoundRobin)) // don't want to unload domains in round robin mode, we don't know how // many tests are left. { lock (daTest) { // make sure no one accesses the app domain at the same time (between here & RunReliabilityTests) if (daTest.RunningCount == 1 && daTest.AppDomain != null) { // only unload when the last test finishes. // unload app domain's async so that we don't block while holding daTest lock. AppDomainUnloadDelegate adu = new AppDomainUnloadDelegate(AppDomain.Unload); adu.BeginInvoke(daTest.AppDomain, null, null); daTest.AppDomain = null; RunCommands(daTest.PostCommands, "post", daTest); } } } } catch (ThreadAbortException) { UnexpectedThreadAbortDebugBreak(); } finally { SignalTestFinished(daTest); } #endif break; } } catch (Exception e) { _logger.WriteToInstrumentationLog(_curTestSet, LoggingLevels.Tests, String.Format("Unexpected exception on StartTestWorker: {0}", e)); } }