public void CaseSkipped(SkipResult result)
{
log.RecordResult(new TestResult(TestCase(result.MethodGroup))
{
DisplayName = result.Name,
Outcome = Map(CaseStatus.Passed),
ComputerName = Environment.MachineName
});
}
public void TestFailed(string name, string type, string method, double duration, string output, string exceptionType, string message, string stackTrace)
{
VsTestResult result = MakeVsTestResult(type, method, duration, output, TestOutcome.Failed);
result.ErrorMessage = message;
result.ErrorStackTrace = stackTrace;
recorder.RecordEnd(result.TestCase, result.Outcome);
recorder.RecordResult(result);
}
protected override bool Visit(ITestFailed testFailed)
{
var result = MakeVsTestResult(testFailed, TestOutcome.Failed);
result.ErrorMessage = testFailed.Message;
result.ErrorStackTrace = testFailed.StackTrace;
recorder.RecordEnd(result.TestCase, result.Outcome);
recorder.RecordResult(result);
return(!cancelledThunk());
}
protected override bool Visit(ITestFailed testFailed)
{
var result = MakeVsTestResult(TestOutcome.Failed, testFailed);
result.ErrorMessage = ExceptionUtility.CombineMessages(testFailed);
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces(testFailed);
recorder.RecordEnd(result.TestCase, result.Outcome);
recorder.RecordResult(result);
return(!cancelledThunk());
}
public void TestFinished(XmlNode resultNode)
{
TestResult ourResult = _testConverter.ConvertTestResult(resultNode);
_recorder.RecordEnd(ourResult.TestCase, ourResult.Outcome);
_recorder.RecordResult(ourResult);
}
public void TestFinished(NUnitTestEventTestCase resultNode)
{
var testId = resultNode.Id;
if (_outputNodes.TryGetValue(testId, out var outputNodes))
{
_outputNodes.Remove(testId);
}
var result = _testConverter.GetVsTestResults(resultNode, outputNodes ?? EmptyNodes);
if (_settings.ConsoleOut == 1 && !string.IsNullOrEmpty(result.ConsoleOutput))
{
_recorder.SendMessage(TestMessageLevel.Informational, result.ConsoleOutput);
}
if (_settings.ConsoleOut == 1 && !string.IsNullOrEmpty(resultNode.ReasonMessage))
{
_recorder.SendMessage(TestMessageLevel.Informational, $"{resultNode.Name}: {resultNode.ReasonMessage}");
}
_recorder.RecordEnd(result.TestCaseResult.TestCase, result.TestCaseResult.Outcome);
foreach (var vsResult in result.TestResults)
{
_recorder.RecordResult(vsResult);
}
if (result.TestCaseResult.Outcome == TestOutcome.Failed && _settings.StopOnError)
{
executor.StopRun();
}
}
private void RunTestsCore(IEnumerable <ITestScript> tests, Predicate <ITestFunction> filter, ILogger logger, IRunContext runContext, ITestExecutionRecorder recorder)
{
var settingsService = runContext.RunSettings.GetSettings(PSycheTestRunSettings.SettingsProviderName) as IPSycheTestSettingsService ?? new PSycheTestSettingsService();
var testCaseMap = new Dictionary <ITestFunction, TestCase>();
var executor = _executorFactory(logger);
executor.OutputDirectory = new DirectoryInfo(runContext.TestRunDirectory);
logger.Info("Test output directory: {0}", executor.OutputDirectory.FullName);
foreach (var module in settingsService.Settings.Modules)
{
logger.Info("Adding module '{0}'", module);
executor.InitialModules.Add(module);
}
executor.TestStarting += (o, e) =>
{
var testCase = _mapper.Map(e.Test);
testCaseMap[e.Test] = testCase;
recorder.RecordStart(testCase);
};
executor.TestEnded += (o, e) =>
{
var testCase = testCaseMap[e.Test];
recorder.RecordEnd(testCase, _mapper.Map(e.Result.Status));
recorder.RecordResult(_mapper.Map(testCase, e.Result));
};
using (_cancellationTokenSource = new CancellationTokenSource())
{
executor.ExecuteAsync(tests, filter, _cancellationTokenSource.Token).Wait(); // Awaiting seems to cause odd behavior, just block instead.
}
_cancellationTokenSource = null;
}
protected override bool Visit(ITestFailed testFailed)
{
var result = MakeVsTestResult(TestOutcome.Failed, testFailed);
if (result != null)
{
result.ErrorMessage = ExceptionUtility.CombineMessages(testFailed);
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces(testFailed);
TryAndReport("RecordResult (Fail)", testFailed.TestCase, () => recorder.RecordResult(result));
}
else
{
logger.LogWarning(testFailed.TestCase, "(Fail) Could not find VS test case for {0} (ID = {1})", testFailed.TestCase.DisplayName, testFailed.TestCase.UniqueID);
}
return(!cancelledThunk());
}
private void LogTestCaseFinished(TestStepFinishedEventArgs e)
{
var testCase = testCaseFactory.GetTestCase(e.Test);
var testResult = testResultFactory.BuildTestResult(e.Test, e.TestStepRun, testCase);
executionRecorder.RecordEnd(testCase, testResult.Outcome);
executionRecorder.RecordResult(testResult);
}
public void TestFinished(XmlNode resultNode)
{
var result = _testConverter.GetVSTestResults(resultNode);
_recorder.RecordEnd(result.TestCaseResult.TestCase, result.TestCaseResult.Outcome);
foreach (var vsResult in result.TestResults)
{
_recorder.RecordResult(vsResult);
}
}
void HandleTestFailed(MessageHandlerArgs <ITestFailed> args)
{
var testFailed = args.Message;
var result = MakeVsTestResult(TestOutcome.Failed, testFailed);
if (result != null)
{
result.ErrorMessage = ExceptionUtility.CombineMessages(testFailed);
result.ErrorStackTrace = ExceptionUtility.CombineStackTraces(testFailed);
TryAndReport("RecordResult (Fail)", testFailed.TestCase, () => recorder.RecordResult(result));
}
else
{
logger.LogWarning(testFailed.TestCase, "(Fail) Could not find VS test case for {0} (ID = {1})", testFailed.TestCase.DisplayName, testFailed.TestCase.UniqueID);
}
HandleCancellation(args);
}
internal void SendTestResults(TestCase test, UnitTestResult[] unitTestResults, DateTimeOffset startTime, DateTimeOffset endTime, ITestExecutionRecorder testExecutionRecorder)
{
if (!(unitTestResults?.Length > 0))
{
return;
}
Guid originalTestCaseId = test.Id;
foreach (var unitTestResult in unitTestResults)
{
if (test == null)
{
continue;
}
// If the result has a TestId set then update the TestCase to have that id
// this is done for data driven results so we can fire unique start/end events for each iteration
test.Id = unitTestResult.TestId == Guid.Empty ? originalTestCaseId : unitTestResult.TestId;
var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings);
if (unitTestResult.DatarowIndex >= 0)
{
testResult.DisplayName = string.Format(CultureInfo.CurrentCulture, Resource.DataDrivenResultDisplayName, test.DisplayName, unitTestResult.DatarowIndex);
}
// Fire a RecordEnd here even though we also fire RecordEnd inside TestMethodInfo
// this is to ensure we fire end events for error cases. This is acceptable because
// vstest ignores multiple end events.
testExecutionRecorder.RecordEnd(test, testResult.Outcome);
if (testResult.Outcome == TestOutcome.Failed)
{
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("MSTestExecutor:Test {0} failed. ErrorMessage:{1}, ErrorStackTrace:{2}.", testResult.TestCase.FullyQualifiedName, testResult.ErrorMessage, testResult.ErrorStackTrace);
this.HasAnyTestFailed = true;
}
try
{
testExecutionRecorder.RecordResult(testResult);
}
catch (TestCanceledException)
{
// Ignore this exception
}
}
// Reset the test id back to the original id in case we changed it above
test.Id = originalTestCaseId;
}
private Task ExecuteTests(IEnumerable <TestCase> tests, ITestExecutionRecorder frameworkHandle, Executor executor, CancellationToken token)
{
return(Task.WhenAll(tests.Select(test => Task.Run(async() =>
{
frameworkHandle.RecordStart(test);
try
{
var result = await executor.Execute(_testDiscoverer.GetTest(test.FullyQualifiedName), token);
frameworkHandle.RecordEnd(test, result.Outcome.ToVsResultOutcome());
frameworkHandle.RecordResult(result.ConvertToVsResult(test));
}
catch (Exception ex)
{
frameworkHandle.RecordEnd(test, TestOutcome.None);
frameworkHandle.RecordResult(new TestResult(test)
{
Outcome = TestOutcome.None,
ErrorMessage = ex.Message,
ErrorStackTrace = ex.StackTrace
});
}
}, token))));
}
public void TestFinished(XmlNode resultNode)
{
bool firstResult = true;
foreach (var vsResult in _testConverter.GetVSTestResults(resultNode))
{
if (firstResult)
{
_recorder.RecordEnd(vsResult.TestCase, vsResult.Outcome);
firstResult = false;
}
_recorder.RecordResult(vsResult);
}
}
void Record(PipeMessage.CaseCompleted result, Action <TestResult> customize = null)
{
var testCase = ToVsTestCase(result.Test);
var testResult = new TestResult(testCase)
{
DisplayName = result.Name,
Duration = result.Duration,
ComputerName = MachineName
};
customize?.Invoke(testResult);
AttachCapturedConsoleOutput(result.Output, testResult);
log.RecordResult(testResult);
}
public void TestFinished(XmlNode resultNode)
{
ICollection <XmlNode> outputNodes;
var testId = resultNode.GetAttribute("id");
if (_outputNodes.TryGetValue(testId, out outputNodes))
{
_outputNodes.Remove(testId);
}
var result = _testConverter.GetVSTestResults(resultNode, outputNodes ?? EmptyNodes);
_recorder.RecordEnd(result.TestCaseResult.TestCase, result.TestCaseResult.Outcome);
foreach (var vsResult in result.TestResults)
{
_recorder.RecordResult(vsResult);
}
}
private void RunTests(ITestExecutionRecorder testExecutionRecorder)
{
TestLauncherResult testLauncherResult = _launcher.Run();
foreach (var allTestStepRun in testLauncherResult.Report.TestPackageRun.AllTestStepRuns)
{
TestCase tc = new TestCase(allTestStepRun.Step.FullName, new Uri(ExecutorUri));
ObjectModel.TestResult testResult = new ObjectModel.TestResult(tc);
testResult.DisplayName = allTestStepRun.Step.FullName;
testResult.ErrorLineNumber = allTestStepRun.Step.CodeLocation.Line;
//testResult.ErrorStackTrace
testResult.StartTime = allTestStepRun.StartTime;
//testResult.ErrorMessage -- why this and messages?
testResult.EndTime = allTestStepRun.EndTime;
//testResult.Attachments = allTestStepRun.TestLog.Attachments; -- can I append my own?
testResult.Duration = allTestStepRun.Result.Duration;
//testResult.Messages =
var testStatus = allTestStepRun.Result.Outcome.Status;
switch (testStatus)
{
case TestStatus.Passed:
testResult.Outcome = ObjectModel.TestOutcome.Passed;
break;
case TestStatus.Failed:
testResult.Outcome = ObjectModel.TestOutcome.Failed;
break;
case TestStatus.Skipped:
testResult.Outcome = ObjectModel.TestOutcome.Skipped;
break;
case TestStatus.Inconclusive: //is this right to use not found?
testResult.Outcome = ObjectModel.TestOutcome.Skipped;
break;
}
testExecutionRecorder.RecordResult(testResult);
}
}
internal void SendTestResults(TestCase test, UnitTestResult[] unitTestResults, DateTimeOffset startTime, DateTimeOffset endTime, ITestExecutionRecorder testExecutionRecorder)
{
if (!(unitTestResults?.Length > 0))
{
return;
}
foreach (var unitTestResult in unitTestResults)
{
if (test == null)
{
continue;
}
var testResult = unitTestResult.ToTestResult(test, startTime, endTime, MSTestSettings.CurrentSettings);
if (unitTestResult.DatarowIndex >= 0)
{
testResult.DisplayName = string.Format(CultureInfo.CurrentCulture, Resource.DataDrivenResultDisplayName, test.DisplayName, unitTestResult.DatarowIndex);
}
testExecutionRecorder.RecordEnd(test, testResult.Outcome);
if (testResult.Outcome == TestOutcome.Failed)
{
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("MSTestExecutor:Test {0} failed. ErrorMessage:{1}, ErrorStackTrace:{2}.", testResult.TestCase.FullyQualifiedName, testResult.ErrorMessage, testResult.ErrorStackTrace);
this.HasAnyTestFailed = true;
}
try
{
testExecutionRecorder.RecordResult(testResult);
}
catch (TestCanceledException)
{
// Ignore this exception
}
}
}
private void RunTest(ITestExecutionRecorder frameworkHandle, string source, string spec = null)
{
var results = appDomainRunner.ExecuteSpecifications(source, spec);
var query = from result in results
from @group in result.Examples
from example in @group.Examples
select example;
foreach (var example in query)
{
var testCase = new TestCase(example.Reason, ExecutorUri, source)
{
CodeFilePath = source
};
frameworkHandle.RecordStart(testCase);
var testResult = new TestResult(testCase)
{
DisplayName = example.Reason,
Duration = new TimeSpan(example.ElapsedTime),
};
if (example.Status == ResultStatus.Error)
{
testResult.Outcome = TestOutcome.Failed;
testResult.ErrorMessage = example.Message;
testResult.ErrorStackTrace = example.StackTrace;
}
if (example.Status == ResultStatus.Success)
{
testResult.Outcome = TestOutcome.Passed;
}
frameworkHandle.RecordEnd(testCase, testResult.Outcome);
frameworkHandle.RecordResult(testResult);
}
}
private static void ExecuteStandardTest(ITestExecutionRecorder recorder, TestCase test, MethodInfo method,
TestContext context, object instance)
{
try
{
recorder.SendMessage(TestMessageLevel.Informational, "Running standard test");
var accessor = CallAccessor.Create(method);
var occurrences = 0;
recorder.RecordResult(DutyCycle(accessor, ++occurrences));
while (!context.Skipped && context.RepeatCount > 0)
{
recorder.RecordResult(DutyCycle(accessor, ++occurrences));
context.RepeatCount--;
}
}
finally
{
context.Dispose();
}
TestResult DutyCycle(IMethodCallAccessor accessor, int occurrence)
{
if (occurrence > 1)
{
var clone = new TestCase
{
DisplayName = $"{test.DisplayName ?? "<No Name>"} #{occurrence}",
ExecutorUri = test.ExecutorUri,
FullyQualifiedName = test.FullyQualifiedName,
Id = test.Id,
LineNumber = test.LineNumber,
LocalExtensionData = test.LocalExtensionData,
CodeFilePath = test.CodeFilePath,
Source = test.Source
};
clone.Traits.AddRange(test.Traits);
test = clone;
}
TestResult testResult = null;
try
{
recorder.SendMessage(TestMessageLevel.Informational, "Starting test");
recorder.RecordStart(test);
context.BeginTest();
var result = ExecuteTestMethod(accessor, instance, context);
context.EndTest();
var outcome = context.Skipped ? TestOutcome.Skipped :
result ? TestOutcome.Passed : TestOutcome.Failed;
recorder.SendMessage(TestMessageLevel.Informational,
$"{test.DisplayName} => {outcome.ToString().ToLowerInvariant()}");
recorder.SendMessage(TestMessageLevel.Informational, "Ending test");
recorder.RecordEnd(test, outcome);
testResult = new TestResult(test)
{
Outcome = outcome
};
if (context.Skipped)
{
testResult.Messages.Add(new TestResultMessage(Constants.Categories.Skipped,
context.SkipReason));
}
var listener = Trace.Listeners.OfType <TestResultTraceListener>().SingleOrDefault();
if (listener != null)
{
testResult.Messages.Add(listener.ToMessage());
}
return(testResult);
}
catch (Exception ex)
{
recorder.SendMessage(TestMessageLevel.Error, ex.Message);
testResult ??= new TestResult(test);
testResult.Messages.Add(new TestResultMessage(Constants.Categories.Failed,
"Test failed because an exception prevented execution."));
testResult.Outcome = TestOutcome.Failed;
testResult.ErrorMessage = ex.Message;
testResult.ErrorStackTrace = ex.StackTrace;
return(testResult);
}
}
}
public void read(IEnumerable <TestCase> testCases, string content)
{
try
{
Dictionary <String, IglooResult> results = new Dictionary <string, IglooResult>();
XmlDocument doc = new XmlDocument();
doc.LoadXml(content);
var testNodes = doc.SelectNodes("//testsuite/testcase");
foreach (XmlNode testNode in testNodes)
{
XmlAttribute nameAttribute = testNode.Attributes["name"];
XmlAttribute classnameAttribute = testNode.Attributes["classname"];
if (nameAttribute != null && !String.IsNullOrWhiteSpace(nameAttribute.Value))
{
string name = nameAttribute.Value;
if (classnameAttribute != null && !String.IsNullOrWhiteSpace(classnameAttribute.Value))
{
name = classnameAttribute.Value + "::" + name;
}
IglooResult result = new IglooResult();
XmlNode failureNode = testNode.SelectSingleNode("failure");
if (failureNode == null)
{
// Success, yay
result.Outcome = TestOutcome.Passed;
}
else
{
result.Outcome = TestOutcome.Failed;
XmlAttribute messageAttribute = failureNode.Attributes["message"];
string message = messageAttribute.Value;
// Need single line option to match the multiline error message
Regex r = new Regex(@"([^(]+)[ ]?\(([0-9]+)\): (.*)$", RegexOptions.Singleline);
Match m = r.Match(message);
if (m.Success)
{
result.File = m.Groups[1].Value;
result.LineNo = Convert.ToInt32(m.Groups[2].Value, CultureInfo.InvariantCulture);
result.ErrorMessage = m.Groups[3].Value;
}
else
{
result.ErrorMessage = message;
}
}
results[name] = result;
}
}
foreach (TestCase test in testCases)
{
var testResult = new TestResult(test);
if (results.ContainsKey(test.FullyQualifiedName))
{
IglooResult result = results[test.FullyQualifiedName];
testResult.Outcome = result.Outcome;
testResult.ErrorStackTrace = result.File + ":" + result.LineNo;
testResult.ErrorMessage = result.ErrorMessage;
}
else
{
testResult.Outcome = TestOutcome.NotFound;
}
testResultSink.RecordResult(testResult);
}
}
catch (Exception e)
{
foreach (TestCase test in testCases)
{
var testResult = new TestResult(test);
testResult.Outcome = TestOutcome.Skipped;
testResult.ErrorMessage = "SnowPlow: Unable to perform unit test: " + e.Message;
testResultSink.RecordResult(testResult);
}
throw;
}
}
private void RunTest(ITestExecutionRecorder frameworkHandle, string source, string spec = null)
{
var results = appDomainRunner.ExecuteSpecifications(source, spec);
var query = from result in results
from @group in result.Examples
from example in @group.Examples
select example;
foreach (var example in query) {
var testCase = new TestCase(example.Reason, ExecutorUri, source) {
CodeFilePath = source
};
frameworkHandle.RecordStart(testCase);
var testResult = new TestResult(testCase) {
DisplayName = example.Reason,
Duration = new TimeSpan(example.ElapsedTime),
};
if (example.Status == ResultStatus.Error) {
testResult.Outcome = TestOutcome.Failed;
testResult.ErrorMessage = example.Message;
testResult.ErrorStackTrace = example.StackTrace;
}
if (example.Status == ResultStatus.Success) {
testResult.Outcome = TestOutcome.Passed;
}
frameworkHandle.RecordEnd(testCase, testResult.Outcome);
frameworkHandle.RecordResult(testResult);
}
}
//----------------------------------------------------------------------------------------------------
private async Task ParseTestRunOutput(
ITestExecutionRecorder recorder,
StreamReader stream,
IEnumerable <TestCase> tests
)
{
var tests_complete = false;
TestCase current_test = null;
var current_message_lines = new List <string>();
do
{
var line = await stream.ReadLineAsync();
if (line == null)
{
break;
}
if (tests_complete)
{
// We have stopped processing tests for some reason.
break;
}
if (line.StartsWith(" "))
{
// A message line for the current test.
current_message_lines.Add(line.TrimStart());
continue;
}
if (line.StartsWith("Test Complete:"))
{
if (current_test == null)
{
LogError("Unexpected test completion");
break;
}
// The current test has finished. Update the status.
var status = line.Substring(15);
var outcome = (status == "passed") ? TestOutcome.Passed : TestOutcome.Failed;
var message = string.Join(Environment.NewLine, current_message_lines);
recorder.RecordResult(
new TestResult(current_test)
{
Outcome = outcome,
ErrorMessage = message
}
);
LogDebug($"Test Complete: {current_test.FullyQualifiedName} ({status})");
current_test = null;
current_message_lines.Clear();
continue;
}
if (current_test != null)
{
// The current test has finished unexpectedly.
LogError($"Unexpected end of test: {current_test.FullyQualifiedName}");
break;
}
if (line.StartsWith("Skip:"))
{
var test_name = line.Substring(6);
recorder.RecordResult(
new TestResult(tests.First(t => t.FullyQualifiedName == test_name))
{
Outcome = TestOutcome.Skipped
}
);
LogDebug($"Test Skipped: {test_name}");
continue;
}
if (line.StartsWith("Test:"))
{
var test_name = line.Substring(6);
current_test = tests.First(t => t.FullyQualifiedName == test_name);
recorder.RecordStart(current_test);
continue;
}
if (line.StartsWith("Running"))
{
// Ignore the first line.
continue;
}
if (line.StartsWith("Complete."))
{
// All done.
tests_complete = true;
continue;
}
// Something has gone wrong. Stop processing.
tests_complete = true;
} while (!stream.EndOfStream);
}
void ITestExecutionRecorder.RecordResult(TestResult testResult) => recorder?.RecordResult(testResult);
