public void Flush(bool retrySucceeded) { foreach (var message in _messages) { // in case of retry succeeded, convert all failure to skip (ignored) if (retrySucceeded && message is TestFailed) { var failed = (TestFailed)message; var reason = new StringBuilder(); reason.AppendLine(String.Join(Environment.NewLine, failed.ExceptionTypes)); reason.AppendLine(String.Join(Environment.NewLine, failed.Messages)); reason.AppendLine(String.Join(Environment.NewLine, failed.StackTraces)); // xunit does not report output if skipped (ignored) tests. // put it as reason instead. reason.AppendLine("===================================================================================="); reason.AppendLine(failed.Output); var skipped = new TestSkipped(failed.Test, reason.ToString()); _innerBus.QueueMessage(skipped.SanitizeXml()); } else { _innerBus.QueueMessage(message.SanitizeXml()); } } }
public void ConvertsITestSkipped() { TestResult testResult = null; var listener = Substitute.For <ITestListener>(); listener.WhenAny(l => l.TestFinished(null)) .Do <TestResult>(result => testResult = result); var visitor = new ResultVisitor(listener); var message = new TestSkipped { TestCase = new TestCase(typeof(string), "Contains"), TestDisplayName = "Display Name", ExecutionTime = 123.45M, Reason = "I forgot how to run" }; visitor.OnMessage(message); Assert.NotNull(testResult); Assert.Same(typeof(string), testResult.FixtureType); Assert.Equal("Contains", testResult.Method.Name); Assert.Equal("Display Name", testResult.Name); Assert.Equal(TestState.Ignored, testResult.State); Assert.Equal(123.45, testResult.TimeSpan.TotalMilliseconds); Assert.Equal(1, testResult.TotalTests); Assert.Equal("I forgot how to run", testResult.Message); }
// making sure all texts are Xml valid public static IMessageSinkMessage SanitizeXml(this IMessageSinkMessage message) { var failed = message as TestFailed; if (failed != null) { return(new TestFailed(failed.Test, failed.ExecutionTime, XmlUtility.Sanitize(failed.Output), failed.ExceptionTypes, failed.Messages == null ? null : failed.Messages.Select(m => XmlUtility.Sanitize(m)).ToArray(), failed.StackTraces, failed.ExceptionParentIndices)); } var skipped = message as TestSkipped; if (skipped != null) { skipped = new TestSkipped(skipped.Test, XmlUtility.Sanitize(skipped.Reason)); skipped.SetOutput(XmlUtility.Sanitize(skipped.Output)); return(skipped); } var passed = message as TestPassed; if (passed != null) { return(new TestPassed(passed.Test, passed.ExecutionTime, XmlUtility.Sanitize(passed.Output))); } return(message); }
public bool QueueMessage(IMessageSinkMessage message) { var result = message as TestResultMessage; if (result != null && String.IsNullOrEmpty(result.Output)) { result.SetOutput(TestTracer.GetTraceString()); } lock (_messages) { var testFailed = result as TestFailed; if (testFailed != null && testFailed.ExceptionTypes.LastOrDefault() == typeof(KuduXunitTestSkippedException).FullName) { TestSkipped = true; message = new TestSkipped(result.Test, testFailed.Messages.LastOrDefault() ?? "unknown"); } _messages.Add(message); } // No way to ask the inner bus if they want to cancel without sending them the message, so // we just go ahead and continue always. return(true); }
public AcceptanceTestRunner(IList <Type> tests, TestPassed passHandler, TestFailed failHandler, TestSkipped skipHandler) { if (passHandler == null || failHandler == null || skipHandler == null) { throw new ArgumentNullException("Test result handlers cannot be null."); } _tests = tests ?? throw new ArgumentNullException("Test collection cannot be null."); _testPassedHandler = passHandler; _testFailedHandler = failHandler; _testSkippedHandler = skipHandler; }
public bool QueueMessage(IMessageSinkMessage message) { var result = message as TestResultMessage; if (result != null && String.IsNullOrEmpty(result.Output)) { result.SetOutput(TestTracer.GetTraceString()); } var testFailed = result as TestFailed; if (testFailed != null && testFailed.ExceptionTypes.LastOrDefault() == typeof(KuduXunitTestSkippedException).FullName) { TestSkipped = true; message = new TestSkipped(result.Test, testFailed.Messages.LastOrDefault() ?? "unknown"); } return(_innerBus.QueueMessage(message.SanitizeXml())); }
protected override async Task <RunSummary> RunTestAsync() { var scenarioFactTestCase = (ScenarioFactTestCase)TestCase; var test = CreateTest(TestCase, DisplayName); var aggregatedResult = new RunSummary(); // Theories are called with required arguments. Keep track of what arguments we already tested so that we can skip those accordingly var testedArguments = new HashSet <object>(); // Each time we find a new theory argument, we will want to restart our Test so that we can collect subsequent test cases bool pendingRestart; do { // Safeguarding against abuse if (testedArguments.Count >= scenarioFactTestCase.TheoryTestCaseLimit) { pendingRestart = false; MessageBus.QueueMessage(new TestSkipped(test, "Theory tests are capped to prevent infinite loops. You can configure a different limit by setting TheoryTestCaseLimit on the Scenario attribute")); aggregatedResult.Aggregate(new RunSummary { Skipped = 1, Total = 1 }); } else { var bufferedMessageBus = new BufferedMessageBus(MessageBus); var stopwatch = Stopwatch.StartNew(); var skipAdditionalTests = false; var testRecorded = false; pendingRestart = false; // By default we dont expect a new restart object? capturedArgument = null; ScenarioContext scenarioContext = null; scenarioContext = new ScenarioContext(scenarioFactTestCase.FactName, async(ScenarioTestCaseDescriptor descriptor) => { // If we're hitting our target test if (descriptor.Name == scenarioFactTestCase.FactName) { testRecorded = true; if (skipAdditionalTests) { pendingRestart = true; // when we discovered more tests after a test completed, allow us to restart scenarioContext.EndScenarioConditionally(); return; } if (descriptor.Argument is not null) { // If we've already received this test case, don't run it again if (testedArguments.Contains(descriptor.Argument)) { return; } testedArguments.Add(descriptor.Argument); capturedArgument = descriptor.Argument; } // At this stage we found our first valid test case, any subsequent test case should issue a restart instead skipAdditionalTests = true; try { await descriptor.Invocation(); } catch (Exception) { // If we caught an exception but we're in a theory, we will want to try for additional test cases if (descriptor.Argument is not null) { pendingRestart = true; } throw; } finally { scenarioContext.IsTargetConclusive = true; } } else { // We may be hitting a shared fact, those need to be invoked as well but not recorded as our primary target if (!scenarioFactTestCase.RunInIsolation || descriptor.Flags.HasFlag(ScenarioTestCaseFlags.Shared)) { await descriptor.Invocation(); } } }); scenarioContext.AutoAbort = scenarioFactTestCase.ExecutionPolicy is ScenarioTestExecutionPolicy.EndAfterConclusion; TestMethodArguments = new object[] { scenarioContext }; RunSummary result; result = await CreateTestRunner(test, bufferedMessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource).RunAsync(); aggregatedResult.Aggregate(result); stopwatch.Stop(); var testInvocationTest = capturedArgument switch { null => CreateTest(TestCase, DisplayName), not null => CreateTest(TestCase, $"{DisplayName} ({capturedArgument})") }; var bufferedMessages = bufferedMessageBus.QueuedMessages; // We should have expected at least one test run. We probably returned before our target test was able to run if (!testRecorded && result.Failed == 0) { bufferedMessageBus.QueueMessage(new TestSkipped(test, scenarioContext.SkippedReason ?? "No applicable tests were able to run")); result = new RunSummary { Skipped = 1, Total = 1 }; } // If we skipped this test, make sure that this is reported accordingly if (scenarioContext.Skipped && !bufferedMessages.OfType <TestSkipped>().Any()) { bufferedMessages = bufferedMessages.Concat(new[] { new TestSkipped(testInvocationTest, scenarioContext.SkippedReason) }); } // If we have indeed skipped this test, make sure that we're not reporting it as passed or failed if (bufferedMessages.OfType <TestSkipped>().Any()) { bufferedMessages = bufferedMessages.Where(x => x is not TestPassed and not TestFailed); } // If we have a failure in post conditions, don't mark this test case as passed if (bufferedMessages.OfType <TestFailed>().Any()) { bufferedMessages = bufferedMessages.Where(x => x is not TestPassed); } var output = string.Join("", bufferedMessages .OfType <ITestOutput>() .Select(x => x.Output)); var duration = (decimal)stopwatch.Elapsed.TotalSeconds; foreach (var queuedMessage in bufferedMessages) { var transformedMessage = queuedMessage switch { TestStarting testStarting => new TestStarting(testInvocationTest), TestSkipped testSkipped => new TestSkipped(testInvocationTest, testSkipped.Reason), TestPassed testPassed => new TestPassed(testInvocationTest, duration, output), TestFailed testFailed => new TestFailed(testInvocationTest, duration, output, testFailed.ExceptionTypes, testFailed.Messages, testFailed.StackTraces, testFailed.ExceptionParentIndices), TestFinished testFinished => new TestFinished(testInvocationTest, duration, output), _ => queuedMessage }; if (!MessageBus.QueueMessage(transformedMessage)) { return(aggregatedResult); } } } }while (pendingRestart); return(aggregatedResult); } } }
public async Task <RunSummary> RunScenarioAsync() { var runSummary = new RunSummary { Total = 1 }; var output = string.Empty; if (!MessageBus.QueueMessage(new TestStarting(Test))) { CancellationTokenSource.Cancel(); } else { AfterTestStarting(); if (!string.IsNullOrEmpty(SkipReason)) { runSummary.Skipped++; if (!MessageBus.QueueMessage(new TestSkipped(Test, SkipReason))) { CancellationTokenSource.Cancel(); } } else { var aggregator = new ExceptionAggregator(Aggregator); if (!aggregator.HasExceptions) { var tuple = await aggregator.RunAsync(() => InvokeTestAsync(aggregator)); runSummary.Time = tuple.Item1; output = tuple.Item2; } var exception = aggregator.ToException(); TestResultMessage testResult; if (exception == null) { testResult = new TestPassed(Test, runSummary.Time, output); } else if (exception is IgnoreException) { testResult = new TestSkipped(Test, exception.Message); runSummary.Skipped++; } else { testResult = new TestFailed(Test, runSummary.Time, output, exception); runSummary.Failed++; } if (!CancellationTokenSource.IsCancellationRequested) { if (!MessageBus.QueueMessage(testResult)) { CancellationTokenSource.Cancel(); } } } Aggregator.Clear(); BeforeTestFinished(); if (Aggregator.HasExceptions) { if (!MessageBus.QueueMessage(new TestCleanupFailure(Test, Aggregator.ToException()))) { CancellationTokenSource.Cancel(); } } } if (!MessageBus.QueueMessage(new TestFinished(Test, runSummary.Time, output))) { CancellationTokenSource.Cancel(); } return(runSummary); }
bool OnTest(XmlNode xml) { var @continue = true; var testCase = FindTestCase(xml.Attributes["type"].Value, xml.Attributes["method"].Value); var timeAttribute = xml.Attributes["time"]; var time = timeAttribute == null ? 0M : Decimal.Parse(timeAttribute.Value, CultureInfo.InvariantCulture); var outputElement = xml.SelectSingleNode("output"); var output = outputElement == null ? String.Empty : outputElement.InnerText; var displayName = xml.Attributes["name"].Value; ITestCaseMessage resultMessage = null; testCaseResults.Total++; testCaseResults.Time += time; switch (xml.Attributes["result"].Value) { case "Pass": resultMessage = new TestPassed(testCase, displayName, time, output); break; case "Fail": { testCaseResults.Failed++; var failure = xml.SelectSingleNode("failure"); resultMessage = new TestFailed(testCase, displayName, time, output, failure.Attributes["exception-type"].Value, failure.SelectSingleNode("message").InnerText, failure.SelectSingleNode("stack-trace").InnerText); break; } case "Skip": testCaseResults.Skipped++; resultMessage = new TestSkipped(testCase, displayName, xml.SelectSingleNode("reason/message").InnerText); break; } if (resultMessage != null) @continue = messageSink.OnMessage(resultMessage) && @continue; @continue = messageSink.OnMessage(new TestFinished(testCase, displayName, time, output)) && @continue; return @continue && TestClassResults.Continue; }
public void ConvertsITestSkipped() { TestResult testResult = null; var listener = Substitute.For<ITestListener>(); listener.WhenAny(l => l.TestFinished(null)) .Do<TestResult>(result => testResult = result); var visitor = new ResultVisitor(listener); var message = new TestSkipped { TestCase = new TestCase(typeof(string), "Contains"), TestDisplayName = "Display Name", ExecutionTime = 123.45M, Reason = "I forgot how to run" }; visitor.OnMessage(message); Assert.NotNull(testResult); Assert.Same(typeof(string), testResult.FixtureType); Assert.Equal("Contains", testResult.Method.Name); Assert.Equal("Display Name", testResult.Name); Assert.Equal(TestState.Ignored, testResult.State); Assert.Equal(123.45, testResult.TimeSpan.TotalMilliseconds); Assert.Equal(1, testResult.TotalTests); Assert.Equal("I forgot how to run", testResult.Message); }