private void RunTest(ITestExecutionRecorder frameworkHandle, string source, string spec = null)
{
var results = appDomainRunner.ExecuteSpecifications(source, spec);
var query = from result in results
from @group in result.Examples
from example in @group.Examples
select example;
foreach (var example in query) {
var testCase = new TestCase(example.Reason, ExecutorUri, source) {
CodeFilePath = source
};
frameworkHandle.RecordStart(testCase);
var testResult = new TestResult(testCase) {
DisplayName = example.Reason,
Duration = new TimeSpan(example.ElapsedTime),
};
if (example.Status == ResultStatus.Error) {
testResult.Outcome = TestOutcome.Failed;
testResult.ErrorMessage = example.Message;
testResult.ErrorStackTrace = example.StackTrace;
}
if (example.Status == ResultStatus.Success) {
testResult.Outcome = TestOutcome.Passed;
}
frameworkHandle.RecordEnd(testCase, testResult.Outcome);
frameworkHandle.RecordResult(testResult);
}
}
private void RunTestsCore(IEnumerable <ITestScript> tests, Predicate <ITestFunction> filter, ILogger logger, IRunContext runContext, ITestExecutionRecorder recorder)
{
var settingsService = runContext.RunSettings.GetSettings(PSycheTestRunSettings.SettingsProviderName) as IPSycheTestSettingsService ?? new PSycheTestSettingsService();
var testCaseMap = new Dictionary <ITestFunction, TestCase>();
var executor = _executorFactory(logger);
executor.OutputDirectory = new DirectoryInfo(runContext.TestRunDirectory);
logger.Info("Test output directory: {0}", executor.OutputDirectory.FullName);
foreach (var module in settingsService.Settings.Modules)
{
logger.Info("Adding module '{0}'", module);
executor.InitialModules.Add(module);
}
executor.TestStarting += (o, e) =>
{
var testCase = _mapper.Map(e.Test);
testCaseMap[e.Test] = testCase;
recorder.RecordStart(testCase);
};
executor.TestEnded += (o, e) =>
{
var testCase = testCaseMap[e.Test];
recorder.RecordEnd(testCase, _mapper.Map(e.Result.Status));
recorder.RecordResult(_mapper.Map(testCase, e.Result));
};
using (_cancellationTokenSource = new CancellationTokenSource())
{
executor.ExecuteAsync(tests, filter, _cancellationTokenSource.Token).Wait(); // Awaiting seems to cause odd behavior, just block instead.
}
_cancellationTokenSource = null;
}
private void ExecuteTestsWithTestRunner(
IEnumerable <TestCase> tests,
IRunContext runContext,
ITestExecutionRecorder testExecutionRecorder,
string source,
IDictionary <string, object> sourceLevelParameters,
UnitTestRunner testRunner)
{
foreach (var currentTest in tests)
{
if (this.cancellationToken != null && this.cancellationToken.Canceled)
{
break;
}
var unitTestElement = currentTest.ToUnitTestElement(source);
testExecutionRecorder.RecordStart(currentTest);
var startTime = DateTimeOffset.Now;
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executing test {0}", unitTestElement.TestMethod.Name);
// Run single test passing test context properties to it.
var tcmProperties = TcmTestPropertiesProvider.GetTcmProperties(currentTest);
var testContextProperties = this.GetTestContextProperties(tcmProperties, sourceLevelParameters);
var unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, testContextProperties);
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed test {0}", unitTestElement.TestMethod.Name);
var endTime = DateTimeOffset.Now;
this.SendTestResults(currentTest, unitTestResult, startTime, endTime, testExecutionRecorder);
}
}
public void TestStarted(XmlNode testNode)
{
TestCase ourCase = _testConverter.GetCachedTestCase(testNode.GetAttribute("id"));
// Simply ignore any TestCase not found in the cache
if (ourCase != null)
{
_recorder.RecordStart(ourCase);
}
}
public void TestStarted(NUnitTestEventStartTest testNode)
{
var ourCase = _testConverter.GetCachedTestCase(testNode.Id);
// Simply ignore any TestCase not found in the cache
if (ourCase != null)
{
_recorder.RecordStart(ourCase);
}
}
protected override bool Visit(ITestStarting testStarting)
{
var vsTestCase = FindTestCase(testStarting.TestCase);
if (vsTestCase != null)
{
recorder.RecordStart(vsTestCase);
}
return(!cancelledThunk());
}
protected override bool Visit(ITestCaseStarting testCaseStarting)
{
var vsTestCase = FindTestCase(testCaseStarting.TestCase);
if (vsTestCase != null)
{
TryAndReport("RecordStart", testCaseStarting.TestCase, () => recorder.RecordStart(vsTestCase));
}
else
{
logger.LogWarning(testCaseStarting.TestCase, "(Starting) Could not find VS test case for {0} (ID = {1})", testCaseStarting.TestCase.DisplayName, testCaseStarting.TestCase.UniqueID);
}
return(!cancelledThunk());
}
void HandleTestCaseStarting(MessageHandlerArgs <ITestCaseStarting> args)
{
var testCaseStarting = args.Message;
var vsTestCase = FindTestCase(testCaseStarting.TestCase);
if (vsTestCase != null)
{
TryAndReport("RecordStart", testCaseStarting.TestCase, () => recorder.RecordStart(vsTestCase));
}
else
{
logger.LogWarning(testCaseStarting.TestCase, "(Starting) Could not find VS test case for {0} (ID = {1})", testCaseStarting.TestCase.DisplayName, testCaseStarting.TestCase.UniqueID);
}
HandleCancellation(args);
}
private void ExecuteTestsWithTestRunner(
IEnumerable <TestCase> tests,
IRunContext runContext,
ITestExecutionRecorder testExecutionRecorder,
string source,
IDictionary <string, object> sourceLevelParameters,
UnitTestRunner testRunner)
{
var testExecutionRecorderWrapper = new TestExecutionRecorderWrapper(testExecutionRecorder);
foreach (var currentTest in tests)
{
if (this.cancellationToken != null && this.cancellationToken.Canceled)
{
break;
}
var unitTestElement = currentTest.ToUnitTestElement(source);
// Fire a RecordStart here even though we also fire RecordStart inside TestMethodInfo
// this is to ensure we fire start events for error cases. This is acceptable because
// vstest ignores multiple start events.
testExecutionRecorder.RecordStart(currentTest);
var startTime = DateTimeOffset.Now;
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
"Executing test {0}",
unitTestElement.TestMethod.Name);
// Run single test passing test context properties to it.
var tcmProperties = TcmTestPropertiesProvider.GetTcmProperties(currentTest);
var testContextProperties = this.GetTestContextProperties(tcmProperties, sourceLevelParameters);
var unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, testExecutionRecorderWrapper, testContextProperties);
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
"Executed test {0}",
unitTestElement.TestMethod.Name);
var endTime = DateTimeOffset.Now;
this.SendTestResults(currentTest, unitTestResult, startTime, endTime, testExecutionRecorder);
}
}
private void RunTest(ITestExecutionRecorder frameworkHandle, string source, string spec = null)
{
var results = appDomainRunner.ExecuteSpecifications(source, spec);
var query = from result in results
from @group in result.Examples
from example in @group.Examples
select example;
foreach (var example in query)
{
var testCase = new TestCase(example.Reason, ExecutorUri, source)
{
CodeFilePath = source
};
frameworkHandle.RecordStart(testCase);
var testResult = new TestResult(testCase)
{
DisplayName = example.Reason,
Duration = new TimeSpan(example.ElapsedTime),
};
if (example.Status == ResultStatus.Error)
{
testResult.Outcome = TestOutcome.Failed;
testResult.ErrorMessage = example.Message;
testResult.ErrorStackTrace = example.StackTrace;
}
if (example.Status == ResultStatus.Success)
{
testResult.Outcome = TestOutcome.Passed;
}
frameworkHandle.RecordEnd(testCase, testResult.Outcome);
frameworkHandle.RecordResult(testResult);
}
}
private Task ExecuteTests(IEnumerable <TestCase> tests, ITestExecutionRecorder frameworkHandle, Executor executor, CancellationToken token)
{
return(Task.WhenAll(tests.Select(test => Task.Run(async() =>
{
frameworkHandle.RecordStart(test);
try
{
var result = await executor.Execute(_testDiscoverer.GetTest(test.FullyQualifiedName), token);
frameworkHandle.RecordEnd(test, result.Outcome.ToVsResultOutcome());
frameworkHandle.RecordResult(result.ConvertToVsResult(test));
}
catch (Exception ex)
{
frameworkHandle.RecordEnd(test, TestOutcome.None);
frameworkHandle.RecordResult(new TestResult(test)
{
Outcome = TestOutcome.None,
ErrorMessage = ex.Message,
ErrorStackTrace = ex.StackTrace
});
}
}, token))));
}
private static void ExecuteStandardTest(ITestExecutionRecorder recorder, TestCase test, MethodInfo method,
TestContext context, object instance)
{
try
{
recorder.SendMessage(TestMessageLevel.Informational, "Running standard test");
var accessor = CallAccessor.Create(method);
var occurrences = 0;
recorder.RecordResult(DutyCycle(accessor, ++occurrences));
while (!context.Skipped && context.RepeatCount > 0)
{
recorder.RecordResult(DutyCycle(accessor, ++occurrences));
context.RepeatCount--;
}
}
finally
{
context.Dispose();
}
TestResult DutyCycle(IMethodCallAccessor accessor, int occurrence)
{
if (occurrence > 1)
{
var clone = new TestCase
{
DisplayName = $"{test.DisplayName ?? "<No Name>"} #{occurrence}",
ExecutorUri = test.ExecutorUri,
FullyQualifiedName = test.FullyQualifiedName,
Id = test.Id,
LineNumber = test.LineNumber,
LocalExtensionData = test.LocalExtensionData,
CodeFilePath = test.CodeFilePath,
Source = test.Source
};
clone.Traits.AddRange(test.Traits);
test = clone;
}
TestResult testResult = null;
try
{
recorder.SendMessage(TestMessageLevel.Informational, "Starting test");
recorder.RecordStart(test);
context.BeginTest();
var result = ExecuteTestMethod(accessor, instance, context);
context.EndTest();
var outcome = context.Skipped ? TestOutcome.Skipped :
result ? TestOutcome.Passed : TestOutcome.Failed;
recorder.SendMessage(TestMessageLevel.Informational,
$"{test.DisplayName} => {outcome.ToString().ToLowerInvariant()}");
recorder.SendMessage(TestMessageLevel.Informational, "Ending test");
recorder.RecordEnd(test, outcome);
testResult = new TestResult(test)
{
Outcome = outcome
};
if (context.Skipped)
{
testResult.Messages.Add(new TestResultMessage(Constants.Categories.Skipped,
context.SkipReason));
}
var listener = Trace.Listeners.OfType <TestResultTraceListener>().SingleOrDefault();
if (listener != null)
{
testResult.Messages.Add(listener.ToMessage());
}
return(testResult);
}
catch (Exception ex)
{
recorder.SendMessage(TestMessageLevel.Error, ex.Message);
testResult ??= new TestResult(test);
testResult.Messages.Add(new TestResultMessage(Constants.Categories.Failed,
"Test failed because an exception prevented execution."));
testResult.Outcome = TestOutcome.Failed;
testResult.ErrorMessage = ex.Message;
testResult.ErrorStackTrace = ex.StackTrace;
return(testResult);
}
}
}
public bool TestStart(string name, string type, string method)
{
recorder.RecordStart(GetTestCase(type, method));
return(!cancelledThunk());
}
protected override bool Visit(ITestStarting testStarting)
{
recorder.RecordStart(testCases[testStarting.TestCase]);
return(!cancelledThunk());
}
private void LogTestCaseStarted(TestStepStartedEventArgs e)
{
var testCase = testCaseFactory.GetTestCase(e.Test);
executionRecorder.RecordStart(testCase);
}
//----------------------------------------------------------------------------------------------------
private async Task ParseTestRunOutput(
ITestExecutionRecorder recorder,
StreamReader stream,
IEnumerable <TestCase> tests
)
{
var tests_complete = false;
TestCase current_test = null;
var current_message_lines = new List <string>();
do
{
var line = await stream.ReadLineAsync();
if (line == null)
{
break;
}
if (tests_complete)
{
// We have stopped processing tests for some reason.
break;
}
if (line.StartsWith(" "))
{
// A message line for the current test.
current_message_lines.Add(line.TrimStart());
continue;
}
if (line.StartsWith("Test Complete:"))
{
if (current_test == null)
{
LogError("Unexpected test completion");
break;
}
// The current test has finished. Update the status.
var status = line.Substring(15);
var outcome = (status == "passed") ? TestOutcome.Passed : TestOutcome.Failed;
var message = string.Join(Environment.NewLine, current_message_lines);
recorder.RecordResult(
new TestResult(current_test)
{
Outcome = outcome,
ErrorMessage = message
}
);
LogDebug($"Test Complete: {current_test.FullyQualifiedName} ({status})");
current_test = null;
current_message_lines.Clear();
continue;
}
if (current_test != null)
{
// The current test has finished unexpectedly.
LogError($"Unexpected end of test: {current_test.FullyQualifiedName}");
break;
}
if (line.StartsWith("Skip:"))
{
var test_name = line.Substring(6);
recorder.RecordResult(
new TestResult(tests.First(t => t.FullyQualifiedName == test_name))
{
Outcome = TestOutcome.Skipped
}
);
LogDebug($"Test Skipped: {test_name}");
continue;
}
if (line.StartsWith("Test:"))
{
var test_name = line.Substring(6);
current_test = tests.First(t => t.FullyQualifiedName == test_name);
recorder.RecordStart(current_test);
continue;
}
if (line.StartsWith("Running"))
{
// Ignore the first line.
continue;
}
if (line.StartsWith("Complete."))
{
// All done.
tests_complete = true;
continue;
}
// Something has gone wrong. Stop processing.
tests_complete = true;
} while (!stream.EndOfStream);
}
void ITestExecutionRecorder.RecordStart(TestCase testCase) => recorder?.RecordStart(testCase);
private void ExecuteTestsWithTestRunner(
IEnumerable <TestCase> tests,
IRunContext runContext,
ITestExecutionRecorder testExecutionRecorder,
string source,
UnitTestRunner testRunner)
{
TestCase test = null;
UnitTestResult[] unitTestResult = null;
var startTime = DateTimeOffset.MinValue;
var endTime = DateTimeOffset.MinValue;
bool filterHasError;
var filterExpression = this.TestMethodFilter.GetFilterExpression(
runContext,
testExecutionRecorder,
out filterHasError);
if (!filterHasError)
{
foreach (var currentTest in tests)
{
// Skip test if not fitting filter criteria.
if (filterExpression != null && filterExpression.MatchTestCase(
currentTest,
(p) => this.TestMethodFilter.PropertyValueProvider(currentTest, p)) == false)
{
continue;
}
// Send previous test result.
this.SendTestResults(test, unitTestResult, startTime, endTime, testExecutionRecorder);
if (this.cancellationToken != null && this.cancellationToken.Canceled)
{
break;
}
var unitTestElement = currentTest.ToUnitTestElement(source);
testExecutionRecorder.RecordStart(currentTest);
startTime = DateTimeOffset.Now;
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
"Executing test {0}",
unitTestElement.TestMethod.Name);
if (unitTestElement.Ignored)
{
unitTestResult = new[] { new UnitTestResult(UnitTestOutcome.Ignored, null) };
}
else
{
// this is done so that appropriate values of testcontext properties are set at source level
// and are merged with session level parameters
var sourceLevelParameters = PlatformServiceProvider.Instance.SettingsProvider.GetProperties(source);
if (this.sessionParameters != null && this.sessionParameters.Count > 0)
{
sourceLevelParameters = sourceLevelParameters.Concat(this.sessionParameters).ToDictionary(x => x.Key, x => x.Value);
}
unitTestResult = testRunner.RunSingleTest(unitTestElement.TestMethod, sourceLevelParameters);
}
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo(
"Executed test {0}",
unitTestElement.TestMethod.Name);
endTime = DateTimeOffset.Now;
test = currentTest;
}
}
IList <string> warnings = null;
try
{
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executing cleanup methods.");
var cleanupResult = testRunner.RunCleanup();
PlatformServiceProvider.Instance.AdapterTraceLogger.LogInfo("Executed cleanup methods.");
if (cleanupResult != null)
{
warnings = cleanupResult.Warnings;
if (unitTestResult?.Length > 0)
{
var lastResult = unitTestResult[unitTestResult.Length - 1];
lastResult.StandardOut += cleanupResult.StandardOut;
lastResult.StandardError += cleanupResult.StandardError;
lastResult.DebugTrace += cleanupResult.DebugTrace;
}
}
}
finally
{
// Send last test result
this.SendTestResults(test, unitTestResult, startTime, endTime, testExecutionRecorder);
}
this.LogWarnings(testExecutionRecorder, warnings);
}
public void Record(PipeMessage.CaseStarted message)
{
var testCase = ToVsTestCase(message.Test);
log.RecordStart(testCase);
}
