protected override ITestContext CreateTestContext(TestCaseMetadata runningTestCase) { var ctx = new TaskoMeterTestContext(runningTestCase.TestCase.ContextName); TaskoMeterEntity taskometer = runningTestCase.TestCase.EntityOfType <TaskoMeterEntity>(); if (taskometer != null) { ctx.WarmupRepetitions = taskometer.WarmupRepetitions; ctx.Repetitions = taskometer.Repetitions; } return(ctx); }
public TestResultEntity PreProcessResults(TestCaseMetadata metadata, TestResultEntity testResult) { var expChessResult = metadata.TestCase.EntityOfType <ExpectedChessTestResultEntity>(); if (expChessResult != null) { XElement xtestResults = testResult.DataElement; try { if (expChessResult.ExitCode.HasValue) { Assert.AreEqual(expChessResult.ExitCode, testResult.ChessExitCode, "Expected Result: ChessExitCode"); } XElement xfinalStats = testResult.DataElement .Elements(XChessNames.ChessResults) .Elements(XChessNames.FinalStats) .SingleOrDefault(); AssertFinalStatValue(xfinalStats, expChessResult.SchedulesRan, XChessNames.ASchedulesRan, "Schedules Ran"); AssertFinalStatValue(xfinalStats, expChessResult.LastThreadCount, XChessNames.ALastThreadCount, "Last Thread Count"); AssertFinalStatValue(xfinalStats, expChessResult.LastExecSteps, XChessNames.ALastExecSteps, "LastExecSteps"); AssertFinalStatValue(xfinalStats, expChessResult.LastHBExecSteps, XChessNames.ALastHBExecSteps, "Last HB Exec Steps"); // If we got here, than all the assertions passed xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.Passed); xtestResults.SetElementValue(XTestResultNames.ResultMessage, "Passed expected MChess test result assertions."); } catch (AssertFailedException ex) { xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.ResultAssertFailure); xtestResults.SetElementValue(XTestResultNames.ResultMessage, ex.Message); } catch (AssertInconclusiveException ex) { xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.ResultInconclusive); xtestResults.SetElementValue(XTestResultNames.ResultMessage, ex.Message); } catch (Exception ex) { xtestResults.SetAttributeValue(XTestResultNames.ATestResultType, TestResultType.Error); xtestResults.SetElementValue(XTestResultNames.ResultMessage, "Expected Chess Result Assertion: " + ex.Message); } } return(testResult); }
/// <summary> /// Runs the test and returns the result xml. /// </summary> /// <returns></returns> public XElement RunTestCase(TestCaseMetadata metadata) { using (ExecuteMChessTask task = new ExecuteMChessTask()) { task.TestCase = metadata.TestCase; AppTaskController.ExecuteTaskInline(task); if (task.Status == AppTaskStatus.Complete) { return(task.XTestResult); } else if (task.Status == AppTaskStatus.Error) { return(TestResultUtil.CreateErrorXTestResult(task.XError)); } else { return(TestResultUtil.CreateErrorXTestResult("The ExecuteMChessTask completed without producing results.")); } } }
internal ManagedTestCase CreateManagedTestCase(TestCaseMetadata runningTestCase) { XElement xmanagedTestMethod = runningTestCase.TestCase.DataElement.Element(XTestCaseNames.ManagedTestMethod); if (xmanagedTestMethod == null) { throw new Exception("Test case xml missing element " + XTestCaseNames.ManagedTestMethod); } // Get the test assembly Assembly testAssembly = Assembly.LoadFrom((string)xmanagedTestMethod.Attribute("assemblyLocation")); // Determine the managed test method string testName = (string)xmanagedTestMethod.Attribute("fullClassName") + "." + (string)xmanagedTestMethod.Attribute("methodName"); int argCount = runningTestCase.TestArgs == null ? 0 : runningTestCase.TestArgs.Values.Length; MethodInfo testMethod = UnitTestsUtil.FindUnitTestMethodByName(testAssembly, testName, argCount); // And the args object[] args = argCount == 0 ? null : UnitTestsUtil.ParseCommandLineArguments(testMethod, runningTestCase.TestArgs.Values); ITestContext context = CreateTestContext(runningTestCase); return(new ManagedTestCase(testMethod, context, args)); }
/// <summary> /// Reads the test case xml from the file. If not specified, reads the xml from Console.In. /// </summary> /// <param name="testCaseFilePath"></param> /// <returns></returns> internal bool LoadTestCase(string testCaseFilePath) { // Read the test case xml XDocument testCaseXDoc = null; if (!String.IsNullOrEmpty(testCaseFilePath)) { try { testCaseXDoc = XDocument.Load(testCaseFilePath); } catch (Exception ex) { Error = "Could not load test case description file."; ErrorEx = ex; return(false); } } else { try { testCaseXDoc = XDocument.Load(Console.In); } catch (Exception ex) { Error = "Could not load test case description from the standard input stream."; ErrorEx = ex; return(false); } } //Validate the xml try { UnitTestingSchemaUtil.ValidateTestCaseXml(testCaseXDoc); } catch (Exception ex) { Error = "Invalid test case xml: " + ex.Message; return(false); } // Create the RunningTestCase instance try { metadata = new TestCaseMetadata(testCaseXDoc.Root); testController = CreateTestController(metadata.TestTypeName); if (testController == null) { return(false); } runner = testController.CreateRunner(); if (runner == null) { Error = testController.GetType().Name + ".CreateRunner didn't return a runner instance."; return(false); } } catch (Exception ex) { Error = "Error with test case."; ErrorEx = ex; return(false); } return(Error == null); }
/// <summary> /// Runs the test and returns the result xml. /// </summary> /// <returns></returns> public XElement RunTestCase(TestCaseMetadata metadata) { ManagedTestCase testCase = null; object testObject = null; try { testCase = Controller.CreateManagedTestCase(metadata); testObject = testCase.CreateNewTestObject(); } catch (Exception ex) { if (ex is TargetInvocationException) { ex = ((TargetInvocationException)ex).InnerException; } throw new Exception("Unit test threw exception while creating test class.", ex); } // Exec the unit test MethodInfo unitTestMethod = testCase.Method; object[] unitTestArgs = testCase.Arguments; try { Invoke(testCase, testObject, unitTestMethod, unitTestArgs, testCase.DisplayNameWithArgs); if (testCase.ExpectedExceptionType != null) { string errMsg = AssertMessagesUtil.FormatAssertionMessage_ExpectedExceptionNotThrown(testCase); return(TestResultUtil.CreateXTestResult(TestResultType.AssertFailure, errMsg, null)); } } catch (Exception ex) { if (ex is TargetInvocationException) { ex = ((TargetInvocationException)ex).InnerException; } Type exType = ex.GetType(); // Detect expected exceptions and handle Assert exceptions if (testCase.ExpectedExceptionType == null || !testCase.ExpectedExceptionType.IsAssignableFrom(exType)) { // Handle special UnitTest exceptions if (ex is ConcurrencyUnitTestException) { // Pre-defined exceptions can just pass up their messages if (ex is AssertFailedException) { return(TestResultUtil.CreateXTestResult(TestResultType.AssertFailure, ex.Message, ex)); } else if (ex is AssertInconclusiveException) { return(TestResultUtil.CreateXTestResult(TestResultType.Inconclusive, ex.Message, ex)); } #if DEBUG // If there's another ex type that we've defined in the framework but haven't handled // by the prev conditions lets warn the developer. Type cutExType = typeof(ConcurrencyUnitTestException); if (exType != cutExType && exType.Assembly.FullName == cutExType.Assembly.FullName) { System.Diagnostics.Trace.TraceWarning("Unhandled ConcurrencyUnitTestException derived type in the CUT assembly: " + exType.Name + ". Custom handling should be added here."); } #endif // If not a built in exception, then use the regular handling below. } // If not an assert, then it's an unexpected exception if (testCase.ExpectedExceptionType == null) { string errMsg = AssertMessagesUtil.FormatAssertionMessage_UnExpectedExceptionThrown(testCase, ex); // Write the exception to the trace log so a debugger can see it. System.Diagnostics.Trace.TraceError(errMsg + Environment.NewLine + "Stack Trace: " + ex.StackTrace); return(TestResultUtil.CreateXTestResult(TestResultType.Exception, errMsg, ex)); } else // Then the exception isn't what's expected { string errMsg = AssertMessagesUtil.FormatAssertionMessage_ExceptionOfWrongTypeThrown(testCase, ex); // Write the exception to the trace log so a debugger can see it. System.Diagnostics.Trace.TraceError(errMsg + Environment.NewLine + "Stack Trace: " + ex.StackTrace); return(TestResultUtil.CreateXTestResult(TestResultType.AssertFailure, errMsg, ex)); } } // else - The exception was expected, so the test succedded, keep going } return(TestResultUtil.CreateXTestResult(TestResultType.Passed, null, null)); }
public TestResultEntity PreProcessResults(TestCaseMetadata metadata, TestResultEntity testResult) { // Nothing to do return(testResult); }
protected virtual ITestContext CreateTestContext(TestCaseMetadata runningTestCase) { var expResult = runningTestCase.TestCase.ExpectedTestResult; return(new TestContext(runningTestCase.TestCase.ContextName, expResult == null ? null : expResult.Key)); }