public void TestFailed(string name, string type, string method, double duration, string output, string exceptionType, string message, string stackTrace)
{
VsTestResult result = MakeVsTestResult(type, method, duration, output, TestOutcome.Failed);
result.ErrorMessage = message;
result.ErrorStackTrace = stackTrace;
recorder.RecordEnd(result.TestCase, result.Outcome);
recorder.RecordResult(result);
}
protected override bool Visit(ITestFailed testFailed)
{
var result = MakeVsTestResult(testFailed, TestOutcome.Failed);
result.ErrorMessage = testFailed.Message;
result.ErrorStackTrace = testFailed.StackTrace;
recorder.RecordEnd(result.TestCase, result.Outcome);
recorder.RecordResult(result);
return(!cancelledThunk());
}
protected override bool Visit(ITestFailed testFailed)
{
var result = MakeVsTestResult(TestOutcome.Failed, testFailed);
result.ErrorMessage = ExceptionUtility.CombineMessages(testFailed);
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces(testFailed);
recorder.RecordEnd(result.TestCase, result.Outcome);
recorder.RecordResult(result);
return(!cancelledThunk());
}
protected override bool Visit(ITestCaseFinished testCaseFinished)
{
var vsTestCase = FindTestCase(testCaseFinished.TestCase);
if (vsTestCase != null)
{
TryAndReport("RecordEnd", testCaseFinished.TestCase, () => recorder.RecordEnd(vsTestCase, TestOutcome.Passed)); // TODO: Don't have an aggregate outcome here!
}
else
{
logger.LogWarning(testCaseFinished.TestCase, "(Finished) Could not find VS test case for {0} (ID = {1})", testCaseFinished.TestCase.DisplayName, testCaseFinished.TestCase.UniqueID);
}
return(!cancelledThunk());
}
private void RunTestsCore(IEnumerable <ITestScript> tests, Predicate <ITestFunction> filter, ILogger logger, IRunContext runContext, ITestExecutionRecorder recorder)
{
var settingsService = runContext.RunSettings.GetSettings(PSycheTestRunSettings.SettingsProviderName) as IPSycheTestSettingsService ?? new PSycheTestSettingsService();
var testCaseMap = new Dictionary <ITestFunction, TestCase>();
var executor = _executorFactory(logger);
executor.OutputDirectory = new DirectoryInfo(runContext.TestRunDirectory);
logger.Info("Test output directory: {0}", executor.OutputDirectory.FullName);
foreach (var module in settingsService.Settings.Modules)
{
logger.Info("Adding module '{0}'", module);
executor.InitialModules.Add(module);
}
executor.TestStarting += (o, e) =>
{
var testCase = _mapper.Map(e.Test);
testCaseMap[e.Test] = testCase;
recorder.RecordStart(testCase);
};
executor.TestEnded += (o, e) =>
{
var testCase = testCaseMap[e.Test];
recorder.RecordEnd(testCase, _mapper.Map(e.Result.Status));
recorder.RecordResult(_mapper.Map(testCase, e.Result));
};
using (_cancellationTokenSource = new CancellationTokenSource())
{
executor.ExecuteAsync(tests, filter, _cancellationTokenSource.Token).Wait(); // Awaiting seems to cause odd behavior, just block instead.
}
_cancellationTokenSource = null;
}
public void TestFinished(XmlNode resultNode)
{
TestResult ourResult = _testConverter.ConvertTestResult(resultNode);
_recorder.RecordEnd(ourResult.TestCase, ourResult.Outcome);
_recorder.RecordResult(ourResult);
}
void HandleTestCaseFinished(MessageHandlerArgs <ITestCaseFinished> args)
{
var testCaseFinished = args.Message;
var vsTestCase = FindTestCase(testCaseFinished.TestCase);
if (vsTestCase != null)
{
TryAndReport("RecordEnd", testCaseFinished.TestCase, () => recorder.RecordEnd(vsTestCase, GetAggregatedTestOutcome(testCaseFinished)));
}
else
{
logger.LogWarning(testCaseFinished.TestCase, "(Finished) Could not find VS test case for {0} (ID = {1})", testCaseFinished.TestCase.DisplayName, testCaseFinished.TestCase.UniqueID);
}
HandleCancellation(args);
}
public void TestFinished(NUnitTestEventTestCase resultNode)
{
var testId = resultNode.Id;
if (_outputNodes.TryGetValue(testId, out var outputNodes))
{
_outputNodes.Remove(testId);
}
var result = _testConverter.GetVsTestResults(resultNode, outputNodes ?? EmptyNodes);
if (_settings.ConsoleOut == 1 && !string.IsNullOrEmpty(result.ConsoleOutput))
{
_recorder.SendMessage(TestMessageLevel.Informational, result.ConsoleOutput);
}
if (_settings.ConsoleOut == 1 && !string.IsNullOrEmpty(resultNode.ReasonMessage))
{
_recorder.SendMessage(TestMessageLevel.Informational, $"{resultNode.Name}: {resultNode.ReasonMessage}");
}
_recorder.RecordEnd(result.TestCaseResult.TestCase, result.TestCaseResult.Outcome);
foreach (var vsResult in result.TestResults)
{
_recorder.RecordResult(vsResult);
}
if (result.TestCaseResult.Outcome == TestOutcome.Failed && _settings.StopOnError)
{
executor.StopRun();
}
}
private void LogTestCaseFinished(TestStepFinishedEventArgs e)
{
var testCase = testCaseFactory.GetTestCase(e.Test);
var testResult = testResultFactory.BuildTestResult(e.Test, e.TestStepRun, testCase);
executionRecorder.RecordEnd(testCase, testResult.Outcome);
executionRecorder.RecordResult(testResult);
}
public void TestFinished(XmlNode resultNode)
{
var result = _testConverter.GetVSTestResults(resultNode);
_recorder.RecordEnd(result.TestCaseResult.TestCase, result.TestCaseResult.Outcome);
foreach (var vsResult in result.TestResults)
{
_recorder.RecordResult(vsResult);
}
}
internal void SendTestResults(TestCase test, UnitTestResult[] unitTestResults, DateTimeOffset startTime, DateTimeOffset endTime, ITestExecutionRecorder testExecutionRecorder)
{
if (!(unitTestResults?.Length > 0))
{
return;
}
Guid originalTestCaseId = test.Id;
foreach (var unitTestResult in unitTestResults)
{
if (test == null)
{
continue;
}
// If the result has a TestId set then update the TestCase to have that id
// this is done for data driven results so we can fire unique start/end events for each iteration
test.Id = unitTestResult.TestId == Guid.Empty ? originalTestCaseId : unitTestResult.TestId;
var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings);
if (unitTestResult.DatarowIndex >= 0)
{
testResult.DisplayName = string.Format(CultureInfo.CurrentCulture, Resource.DataDrivenResultDisplayName, test.DisplayName, unitTestResult.DatarowIndex);
}
// Fire a RecordEnd here even though we also fire RecordEnd inside TestMethodInfo
// this is to ensure we fire end events for error cases. This is acceptable because
// vstest ignores multiple end events.
testExecutionRecorder.RecordEnd(test, testResult.Outcome);
if (testResult.Outcome == TestOutcome.Failed)
{
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("MSTestExecutor:Test {0} failed. ErrorMessage:{1}, ErrorStackTrace:{2}.", testResult.TestCase.FullyQualifiedName, testResult.ErrorMessage, testResult.ErrorStackTrace);
this.HasAnyTestFailed = true;
}
try
{
testExecutionRecorder.RecordResult(testResult);
}
catch (TestCanceledException)
{
// Ignore this exception
}
}
// Reset the test id back to the original id in case we changed it above
test.Id = originalTestCaseId;
}
public void TestFinished(XmlNode resultNode)
{
bool firstResult = true;
foreach (var vsResult in _testConverter.GetVSTestResults(resultNode))
{
if (firstResult)
{
_recorder.RecordEnd(vsResult.TestCase, vsResult.Outcome);
firstResult = false;
}
_recorder.RecordResult(vsResult);
}
}
private Task ExecuteTests(IEnumerable <TestCase> tests, ITestExecutionRecorder frameworkHandle, Executor executor, CancellationToken token)
{
return(Task.WhenAll(tests.Select(test => Task.Run(async() =>
{
frameworkHandle.RecordStart(test);
try
{
var result = await executor.Execute(_testDiscoverer.GetTest(test.FullyQualifiedName), token);
frameworkHandle.RecordEnd(test, result.Outcome.ToVsResultOutcome());
frameworkHandle.RecordResult(result.ConvertToVsResult(test));
}
catch (Exception ex)
{
frameworkHandle.RecordEnd(test, TestOutcome.None);
frameworkHandle.RecordResult(new TestResult(test)
{
Outcome = TestOutcome.None,
ErrorMessage = ex.Message,
ErrorStackTrace = ex.StackTrace
});
}
}, token))));
}
public void TestFinished(XmlNode resultNode)
{
ICollection <XmlNode> outputNodes;
var testId = resultNode.GetAttribute("id");
if (_outputNodes.TryGetValue(testId, out outputNodes))
{
_outputNodes.Remove(testId);
}
var result = _testConverter.GetVSTestResults(resultNode, outputNodes ?? EmptyNodes);
_recorder.RecordEnd(result.TestCaseResult.TestCase, result.TestCaseResult.Outcome);
foreach (var vsResult in result.TestResults)
{
_recorder.RecordResult(vsResult);
}
}
internal void SendTestResults(TestCase test, UnitTestResult[] unitTestResults, DateTimeOffset startTime, DateTimeOffset endTime, ITestExecutionRecorder testExecutionRecorder)
{
if (!(unitTestResults?.Length > 0))
{
return;
}
foreach (var unitTestResult in unitTestResults)
{
if (test == null)
{
continue;
}
var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings);
if (unitTestResult.DatarowIndex >= 0)
{
testResult.DisplayName = string.Format(CultureInfo.CurrentCulture, Resource.DataDrivenResultDisplayName, test.DisplayName, unitTestResult.DatarowIndex);
}
testExecutionRecorder.RecordEnd(test, testResult.Outcome);
if (testResult.Outcome == TestOutcome.Failed)
{
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("MSTestExecutor:Test {0} failed. ErrorMessage:{1}, ErrorStackTrace:{2}.", testResult.TestCase.FullyQualifiedName, testResult.ErrorMessage, testResult.ErrorStackTrace);
this.HasAnyTestFailed = true;
}
try
{
testExecutionRecorder.RecordResult(testResult);
}
catch (TestCanceledException)
{
// Ignore this exception
}
}
}
private void RunTest(ITestExecutionRecorder frameworkHandle, string source, string spec = null)
{
var results = appDomainRunner.ExecuteSpecifications(source, spec);
var query = from result in results
from @group in result.Examples
from example in @group.Examples
select example;
foreach (var example in query)
{
var testCase = new TestCase(example.Reason, ExecutorUri, source)
{
CodeFilePath = source
};
frameworkHandle.RecordStart(testCase);
var testResult = new TestResult(testCase)
{
DisplayName = example.Reason,
Duration = new TimeSpan(example.ElapsedTime),
};
if (example.Status == ResultStatus.Error)
{
testResult.Outcome = TestOutcome.Failed;
testResult.ErrorMessage = example.Message;
testResult.ErrorStackTrace = example.StackTrace;
}
if (example.Status == ResultStatus.Success)
{
testResult.Outcome = TestOutcome.Passed;
}
frameworkHandle.RecordEnd(testCase, testResult.Outcome);
frameworkHandle.RecordResult(testResult);
}
}
private static void ExecuteStandardTest(ITestExecutionRecorder recorder, TestCase test, MethodInfo method,
TestContext context, object instance)
{
try
{
recorder.SendMessage(TestMessageLevel.Informational, "Running standard test");
var accessor = CallAccessor.Create(method);
var occurrences = 0;
recorder.RecordResult(DutyCycle(accessor, ++occurrences));
while (!context.Skipped && context.RepeatCount > 0)
{
recorder.RecordResult(DutyCycle(accessor, ++occurrences));
context.RepeatCount--;
}
}
finally
{
context.Dispose();
}
TestResult DutyCycle(IMethodCallAccessor accessor, int occurrence)
{
if (occurrence > 1)
{
var clone = new TestCase
{
DisplayName = $"{test.DisplayName ?? "<No Name>"} #{occurrence}",
ExecutorUri = test.ExecutorUri,
FullyQualifiedName = test.FullyQualifiedName,
Id = test.Id,
LineNumber = test.LineNumber,
LocalExtensionData = test.LocalExtensionData,
CodeFilePath = test.CodeFilePath,
Source = test.Source
};
clone.Traits.AddRange(test.Traits);
test = clone;
}
TestResult testResult = null;
try
{
recorder.SendMessage(TestMessageLevel.Informational, "Starting test");
recorder.RecordStart(test);
context.BeginTest();
var result = ExecuteTestMethod(accessor, instance, context);
context.EndTest();
var outcome = context.Skipped ? TestOutcome.Skipped :
result ? TestOutcome.Passed : TestOutcome.Failed;
recorder.SendMessage(TestMessageLevel.Informational,
$"{test.DisplayName} => {outcome.ToString().ToLowerInvariant()}");
recorder.SendMessage(TestMessageLevel.Informational, "Ending test");
recorder.RecordEnd(test, outcome);
testResult = new TestResult(test)
{
Outcome = outcome
};
if (context.Skipped)
{
testResult.Messages.Add(new TestResultMessage(Constants.Categories.Skipped,
context.SkipReason));
}
var listener = Trace.Listeners.OfType <TestResultTraceListener>().SingleOrDefault();
if (listener != null)
{
testResult.Messages.Add(listener.ToMessage());
}
return(testResult);
}
catch (Exception ex)
{
recorder.SendMessage(TestMessageLevel.Error, ex.Message);
testResult ??= new TestResult(test);
testResult.Messages.Add(new TestResultMessage(Constants.Categories.Failed,
"Test failed because an exception prevented execution."));
testResult.Outcome = TestOutcome.Failed;
testResult.ErrorMessage = ex.Message;
testResult.ErrorStackTrace = ex.StackTrace;
return(testResult);
}
}
}
private void RunTest(ITestExecutionRecorder frameworkHandle, string source, string spec = null)
{
var results = appDomainRunner.ExecuteSpecifications(source, spec);
var query = from result in results
from @group in result.Examples
from example in @group.Examples
select example;
foreach (var example in query) {
var testCase = new TestCase(example.Reason, ExecutorUri, source) {
CodeFilePath = source
};
frameworkHandle.RecordStart(testCase);
var testResult = new TestResult(testCase) {
DisplayName = example.Reason,
Duration = new TimeSpan(example.ElapsedTime),
};
if (example.Status == ResultStatus.Error) {
testResult.Outcome = TestOutcome.Failed;
testResult.ErrorMessage = example.Message;
testResult.ErrorStackTrace = example.StackTrace;
}
if (example.Status == ResultStatus.Success) {
testResult.Outcome = TestOutcome.Passed;
}
frameworkHandle.RecordEnd(testCase, testResult.Outcome);
frameworkHandle.RecordResult(testResult);
}
}
void ITestExecutionRecorder.RecordEnd(TestCase testCase, TestOutcome outcome)
{
recorder?.RecordEnd(testCase, outcome);
}
