public virtual TestRunState Run(IEnumerable <ITestCase> testCases = null, TestRunState initialRunState = TestRunState.NoTests)
{
try
{
if (testCases != null)
{
testCases = testCases.ToList();
}
var visitor = new ResultVisitor(testListener)
{
TestRunState = initialRunState
};
toDispose.Push(visitor);
frontController.Run(testCases, visitor);
visitor.Finished.WaitOne();
return(visitor.TestRunState);
}
catch (Exception ex)
{
testListener.WriteLine("Error during test execution:\r\n" + ex, Category.Error);
return(TestRunState.Error);
}
}
public virtual TestRunState Run(IEnumerable <ITestCase> testCases = null, TestRunState initialRunState = TestRunState.NoTests)
{
try
{
if (testCases != null)
{
testCases = testCases.ToList();
}
var visitor = new ResultVisitor(testListener)
{
TestRunState = initialRunState
};
toDispose.Push(visitor);
if (testCases == null)
{
xunit.Run(visitor, new XunitDiscoveryOptions(configuration), new XunitExecutionOptions(configuration));
}
else
{
xunit.Run(testCases, visitor, new XunitExecutionOptions(configuration));
}
visitor.Finished.WaitOne();
return(visitor.TestRunState);
}
catch (Exception ex)
{
testListener.WriteLine("Error during test execution:\r\n" + ex, Category.Error);
return(TestRunState.Error);
}
}
TestRunState ITdNetTestRunner.RunNamespace(ITestListener listener, Assembly assembly, string ns)
{
try
{
using (ExecutorWrapper wrapper = new ExecutorWrapper(new Uri(assembly.CodeBase).LocalPath, null, false))
{
TdNetLogger logger = new TdNetLogger(listener, assembly);
TestRunner runner = new TestRunner(wrapper, logger);
TestRunState runState = TestRunState.NoTests;
foreach (Type type in assembly.GetExportedTypes())
{
if (ns == null || type.Namespace == ns)
{
runState = TestResultMapper.Merge(runState, RunClass(runner, type));
}
}
return(runState);
}
}
catch (ArgumentException)
{
return(TestRunState.NoTests);
}
}
public void GetWorkflowDescriptionTest_Empty()
{
var run = new WorkflowRun();
var runState = new TestRunState(new WorkflowMetadata(), run);
Assert.That(runState.GetSafeWorkflowDescription(), Is.StringContaining(run.Id.ToString()));
}
public virtual TestRunState Run(IEnumerable<ITestCase> testCases = null, TestRunState initialRunState = TestRunState.NoTests)
{
try
{
if (testCases != null)
testCases = testCases.ToList();
var visitor = new ResultVisitor(testListener) { TestRunState = initialRunState };
toDispose.Push(visitor);
if (testCases == null)
xunit.Run(visitor, new XunitDiscoveryOptions(), new XunitExecutionOptions());
else
xunit.Run(testCases, visitor, new XunitExecutionOptions());
visitor.Finished.WaitOne();
return visitor.TestRunState;
}
catch (Exception ex)
{
testListener.WriteLine("Error during test execution:\r\n" + ex, Category.Error);
return TestRunState.Error;
}
}
public async Task <TestRunResponse> StartTestRunAsync(TestCaseInfo testCase, TestRunState state)
{
if (_session.State == TestRunSessionState.NotStarted)
{
await SetSessionStateAsync(TestRunSessionState.Running);
}
_logger.Debug($"Start testing test case {testCase.AzureTestCaseId}, State: {state}");
if (testCase.TestRunId == 0)
{
var testRunResponse = await _client.InsertTestRunAsync(
_session.Id,
new TestRunRequest()
{
TestCaseId = testCase.Id,
StartTime = DateTime.Now,
State = state,
});
testCase.TestRunId = testRunResponse.Id;
return(testRunResponse);
}
else
{
var patch = new JsonPatchDocument <TestRunRequest>();
patch.Add(r => r.StartTime, DateTime.Now);
patch.Add(r => r.State, state);
return(await _client.PatchTestRunAsync(_session.Id, testCase.TestRunId, patch));
}
}
public virtual TestRunState Run(IReadOnlyList <ITestCase> testCases = null, TestRunState initialRunState = TestRunState.NoTests)
{
try
{
if (testCases == null)
{
testCases = Discover();
}
var visitor = new ResultVisitor(testListener, testCases.Count)
{
TestRunState = initialRunState
};
toDispose.Push(visitor);
var executionOptions = TestFrameworkOptions.ForExecution(configuration);
xunit.RunTests(testCases, visitor, executionOptions);
visitor.Finished.WaitOne();
return(visitor.TestRunState);
}
catch (Exception ex)
{
testListener.WriteLine("Error during test execution:\r\n" + ex, Category.Error);
return(TestRunState.Error);
}
}
private Task <TestRunResponse> SetTestRunStateAsync(TestCaseInfo testCase, TestRunState state)
{
var patch = new JsonPatchDocument <TestRunRequest>();
patch.Add(r => r.State, state);
_logger.Debug($"Updating test case state: {testCase.AzureTestCaseId} => {state}");
return(_client.PatchTestRunAsync(_session.Id, testCase.TestRunId, patch));
}
void HandleTestSkipped(MessageHandlerArgs<ITestSkipped> args)
{
if (TestRunState == TestRunState.NoTests)
TestRunState = TestRunState.Success;
var testSkipped = args.Message;
var testResult = testSkipped.ToTdNetTestResult(TestState.Ignored, totalTests);
testResult.Message = testSkipped.Reason;
TestListener.TestFinished(testResult);
}
public void FailureSetsStateToFailed(TestRunState initialState)
{
var listener = Substitute.For <ITestListener>();
var visitor = new ResultVisitor(listener)
{
TestRunState = initialState
};
visitor.OnMessage(new TestFailed());
Assert.Equal(TestRunState.Failure, visitor.TestRunState);
}
public void Skip_StaysInCurrentState(TestRunState initialState)
{
var listener = Substitute.For <ITestListener>();
var visitor = new ResultVisitor(listener)
{
TestRunState = initialState
};
visitor.OnMessage(new TestSkipped());
Assert.Equal(initialState, visitor.TestRunState);
}
void HandleTestPassed(MessageHandlerArgs<ITestPassed> args)
{
if (TestRunState == TestRunState.NoTests)
TestRunState = TestRunState.Success;
var testPassed = args.Message;
var testResult = testPassed.ToTdNetTestResult(TestState.Passed, totalTests);
TestListener.TestFinished(testResult);
WriteOutput(testPassed.Test.DisplayName, testPassed.Output);
}
public static void Success_StaysInCurrentState(TestRunState initialState)
{
var listener = Substitute.For <ITestListener>();
var visitor = new ResultVisitor(listener, 42)
{
TestRunState = initialState
};
visitor.OnMessage(Substitute.For <ITestPassed>());
Assert.Equal(initialState, visitor.TestRunState);
}
public static void FailureSetsStateToFailed(TestRunState initialState)
{
var listener = Substitute.For <ITestListener>();
var visitor = new ResultSink(listener, 42)
{
TestRunState = initialState
};
visitor.OnMessageWithTypes(Mocks.TestFailed(typeof(object), "GetHashCode"), null);
Assert.Equal(TestRunState.Failure, visitor.TestRunState);
}
public static void Skip_StaysInCurrentState(TestRunState initialState)
{
var listener = Substitute.For <ITestListener>();
var visitor = new ResultSink(listener, 42)
{
TestRunState = initialState
};
visitor.OnMessageWithTypes(Substitute.For <ITestSkipped>(), null);
Assert.Equal(initialState, visitor.TestRunState);
}
void HandleTestFailed(MessageHandlerArgs<ITestFailed> args)
{
TestRunState = TestRunState.Failure;
var testFailed = args.Message;
var testResult = testFailed.ToTdNetTestResult(TestState.Failed, totalTests);
testResult.Message = ExceptionUtility.CombineMessages(testFailed);
testResult.StackTrace = ExceptionUtility.CombineStackTraces(testFailed);
TestListener.TestFinished(testResult);
WriteOutput(testFailed.Test.DisplayName, testFailed.Output);
}
public virtual TestRunState RunClass(Type type, TestRunState initialRunState = TestRunState.NoTests)
{
var state = Run(Discover(type), initialRunState);
foreach (MemberInfo memberInfo in type.GetMembers())
{
Type childType = memberInfo as Type;
if (childType != null)
state = RunClass(childType, state);
}
return state;
}
void HandleTestSkipped(MessageHandlerArgs <ITestSkipped> args)
{
if (TestRunState == TestRunState.NoTests)
{
TestRunState = TestRunState.Success;
}
var testSkipped = args.Message;
var testResult = testSkipped.ToTdNetTestResult(TestState.Ignored, totalTests);
testResult.Message = testSkipped.Reason;
TestListener.TestFinished(testResult);
}
void HandleTestFailed(MessageHandlerArgs <ITestFailed> args)
{
TestRunState = TestRunState.Failure;
var testFailed = args.Message;
var testResult = testFailed.ToTdNetTestResult(TestState.Failed, totalTests);
testResult.Message = ExceptionUtility.CombineMessages(testFailed);
testResult.StackTrace = ExceptionUtility.CombineStackTraces(testFailed);
TestListener.TestFinished(testResult);
WriteOutput(testFailed.Test.DisplayName, testFailed.Output);
}
void HandleTestPassed(MessageHandlerArgs <ITestPassed> args)
{
if (TestRunState == TestRunState.NoTests)
{
TestRunState = TestRunState.Success;
}
var testPassed = args.Message;
var testResult = testPassed.ToTdNetTestResult(TestState.Passed, totalTests);
TestListener.TestFinished(testResult);
WriteOutput(testPassed.Test.DisplayName, testPassed.Output);
}
public void OnRunEnd()
{
if (testResults.Count == 0) return;
var failure = false;
foreach (var testResult in testResults)
{
testListener.TestFinished(testResult);
failure |= testResult.State == TestState.Failed;
}
testRunState = failure ? TestRunState.Failure : TestRunState.Success;
}
public static TestRunState Merge(TestRunState current, TestRunState toMerge)
{
if (toMerge == TestRunState.Success)
{
if (current == TestRunState.NoTests)
return TestRunState.Success;
}
else if (toMerge == TestRunState.Failure)
{
return TestRunState.Failure;
}
return current;
}
static void AssertFailure(
ITestListener listener,
TestRunState testRunState,
string messageType)
{
Assert.Equal(TestRunState.Failure, testRunState);
var testResult = listener.Captured(x => x.TestFinished(null)).Arg <TestResult>();
Assert.Equal($"*** {messageType} ***", testResult.Name);
Assert.Equal(TestState.Failed, testResult.State);
Assert.Equal(1, testResult.TotalTests);
Assert.Equal("ExceptionType : This is my message \t\r\n", testResult.Message);
Assert.Equal("Line 1\r\nLine 2\r\nLine 3", testResult.StackTrace);
}
protected override bool Visit(ITestPassed passed)
{
if (TestRunState == TestRunState.NoTests)
{
TestRunState = TestRunState.Success;
}
var testResult = passed.ToTdNetTestResult(TestState.Passed);
TestListener.TestFinished(testResult);
WriteOutput(passed.TestDisplayName, passed.Output);
return(true);
}
public void GetWorkflowDescriptionTest_MissingActivity()
{
var workflow = new Workflow()
{
Name = "wfName"
};
var workflowRun = new WorkflowRun()
{
WorkflowBeingRun = workflow
};
var runState = new TestRunState(new WorkflowMetadata(workflow), workflowRun);
Assert.That(runState.GetSafeWorkflowDescription(), Is.StringContaining(workflowRun.Id.ToString()));
}
protected override bool Visit(ITestSkipped skipped)
{
if (TestRunState == TestRunState.NoTests)
{
TestRunState = TestRunState.Success;
}
var testResult = skipped.ToTdNetTestResult(TestState.Ignored);
testResult.Message = skipped.Reason;
TestListener.TestFinished(testResult);
return(true);
}
protected override bool Visit(ITestFailed failed)
{
TestRunState = TestRunState.Failure;
var testResult = failed.ToTdNetTestResult(TestState.Failed);
testResult.Message = ExceptionUtility.CombineMessages(failed);
testResult.StackTrace = ExceptionUtility.CombineStackTraces(failed);
TestListener.TestFinished(testResult);
WriteOutput(failed.TestDisplayName, failed.Output);
return(true);
}
protected override bool Visit(ITestFailed failed)
{
TestRunState = TestRunState.Failure;
TestResult testResult = failed.ToTdNetTestResult(TestState.Failed);
testResult.Message = failed.Message;
testResult.StackTrace = failed.StackTrace;
TestListener.TestFinished(testResult);
//WriteOutput(name, output);
return(true);
}
public virtual TestRunState RunClass(Type type, TestRunState initialRunState = TestRunState.NoTests)
{
var state = Run(Discover(type), initialRunState);
foreach (var memberInfo in type.GetMembers())
{
var childType = memberInfo as Type;
if (childType != null)
{
state = RunClass(childType, state);
}
}
return(state);
}
public static TestRunState Merge(TestRunState current, TestRunState toMerge)
{
if (toMerge == TestRunState.Success)
{
if (current == TestRunState.NoTests)
{
return(TestRunState.Success);
}
}
else if (toMerge == TestRunState.Failure)
{
return(TestRunState.Failure);
}
return(current);
}
public static async void FailureSetsStateToFailed(TestRunState initialState)
{
var listener = Substitute.For <ITestListener>();
await using var sink = new ResultSink(listener, 42)
{
TestRunState = initialState
};
sink.OnMessage(TestData.TestClassStarting());
sink.OnMessage(TestData.TestMethodStarting());
sink.OnMessage(TestData.TestStarting());
sink.OnMessage(TestData.TestFailed());
Assert.Equal(TestRunState.Failure, sink.TestRunState);
}
public virtual TestRunState Run(IEnumerable<ITestCase> testCases, TestRunState initialRunState = TestRunState.NoTests)
{
try
{
var visitor = new ResultVisitor(testListener) { TestRunState = initialRunState };
frontController.Run(testCases.ToList(), visitor);
visitor.Finished.WaitOne();
return visitor.TestRunState;
}
catch (Exception ex)
{
testListener.WriteLine("Error during test execution:\r\n" + ex, Category.Error);
return TestRunState.Error;
}
}
void ReportError(string messageType, IFailureInformation failureInfo)
{
TestRunState = TestRunState.Failure;
var testResult = new TestResult
{
Name = $"*** {messageType} ***",
State = TestState.Failed,
TimeSpan = TimeSpan.Zero,
TotalTests = 1,
Message = ExceptionUtility.CombineMessages(failureInfo),
StackTrace = ExceptionUtility.CombineStackTraces(failureInfo)
};
TestListener.TestFinished(testResult);
}
void ReportError(string messageType, IFailureInformation failureInfo)
{
TestRunState = TestRunState.Failure;
var testResult = new TestResult
{
Name = String.Format("*** {0} ***", messageType),
State = TestState.Failed,
TimeSpan = TimeSpan.Zero,
TotalTests = 1,
Message = ExceptionUtility.CombineMessages(failureInfo),
StackTrace = ExceptionUtility.CombineStackTraces(failureInfo)
};
TestListener.TestFinished(testResult);
}
public static TestRunState RunClassWithInnerTypes(TestRunner runner, Type type)
{
TestRunState result = RunClass(runner, type);
foreach (MemberInfo memberInfo in type.GetMembers())
{
Type childType = memberInfo as Type;
if (childType != null)
{
result = TestResultMapper.Merge(result, RunClassWithInnerTypes(runner, childType));
}
}
return(result);
}
public static async void Success_StaysInCurrentState(TestRunState initialState)
{
var listener = Substitute.For <ITestListener>();
await using var sink = new ResultSink(listener, 42)
{
TestRunState = initialState
};
sink.OnMessage(TestData.TestClassStarting());
sink.OnMessage(TestData.TestMethodStarting());
sink.OnMessage(TestData.TestStarting());
sink.OnMessage(TestData.TestPassed());
Assert.Equal(initialState, sink.TestRunState);
}
public void EffectiveUserIsTriggeringUser()
{
var triggerer = new UserAccount();
triggerer.Name = "foo" + Guid.NewGuid().ToString();
triggerer.Save();
using (var setUser = new SetUser(triggerer))
{
var run = new WorkflowRunDeferred(new Workflow());
var runState = new TestRunState(new WorkflowMetadata(), run);
Assert.That(runState.EffectiveSecurityContext.Identity, Is.Not.Null);
Assert.That(runState.EffectiveSecurityContext.Identity.Id, Is.EqualTo(triggerer.Id));
Assert.That(runState.EffectiveSecurityContext.SecondaryIdentity, Is.Null);
}
}
public void OnRunEnd()
{
if (testResults.Count == 0)
{
return;
}
bool failure = false;
foreach (var testResult in testResults)
{
testListener.TestFinished(testResult);
failure |= testResult.State == TestState.Failed;
}
testRunState = failure ? TestRunState.Failure : TestRunState.Success;
}
void ReportError(
string messageType,
_IErrorMetadata errorMetadata)
{
TestRunState = TestRunState.Failure;
var testResult = new TestResult
{
Name = $"*** {messageType} ***",
State = TestState.Failed,
TimeSpan = TimeSpan.Zero,
TotalTests = 1,
Message = ExceptionUtility.CombineMessages(errorMetadata),
StackTrace = ExceptionUtility.CombineStackTraces(errorMetadata)
};
TestListener.TestFinished(testResult);
}
public virtual TestRunState Run(IReadOnlyList<ITestCase> testCases = null, TestRunState initialRunState = TestRunState.NoTests)
{
try
{
if (testCases == null)
testCases = Discover();
var resultSink = new ResultSink(testListener, testCases.Count) { TestRunState = initialRunState };
toDispose.Push(resultSink);
var executionOptions = TestFrameworkOptions.ForExecution(configuration);
xunit.RunTests(testCases, resultSink, executionOptions);
resultSink.Finished.WaitOne();
return resultSink.TestRunState;
}
catch (Exception ex)
{
testListener.WriteLine("Error during test execution:\r\n" + ex, Category.Error);
return TestRunState.Error;
}
}
public virtual TestRunState RunMethod(MethodInfo method, TestRunState initialRunState = TestRunState.NoTests)
{
var testCases = Discover(method.ReflectedType).Where(tc =>
{
var methodInfo = tc.GetMethod();
if (methodInfo == method)
return true;
if (methodInfo.IsGenericMethod)
return methodInfo.GetGenericMethodDefinition() == method;
return false;
}).ToList();
return Run(testCases, initialRunState);
}
public virtual TestRunState RunMethod(MethodInfo method, TestRunState initialRunState = TestRunState.NoTests)
{
var testCases = Discover(method.ReflectedType).Where(tc => tc.GetMethod() == method);
return Run(testCases, initialRunState);
}
private void Initialize(bool create = true)
{
if (create)
{
_expectedResult = TestCaseResult.Passed;
_runState = TestRunState.NotStarted;
this.LogMessages = new ObservableCollection<string>();
_postValues = new ObservableCollection<PostRunPairs>();
}
}
public void FailureSetsStateToFailed(TestRunState initialState)
{
var listener = Substitute.For<ITestListener>();
var visitor = new ResultVisitor(listener) { TestRunState = initialState };
visitor.OnMessage(new TestFailed());
Assert.Equal(TestRunState.Failure, visitor.TestRunState);
}
public void Success_StaysInCurrentState(TestRunState initialState)
{
var listener = Substitute.For<ITestListener>();
var visitor = new ResultVisitor(listener) { TestRunState = initialState };
visitor.OnMessage(new TestPassed());
Assert.Equal(initialState, visitor.TestRunState);
}
