public async Task <TestResultCollection> Run(ITestCategory category,
CancellationToken cancellationToken)
{
TestResultCollection result;
var runners = GetTestRunners(category);
if (runners.Count == 1)
{
result = await Run(category, runners [0], cancellationToken);
AddWarnings(result);
return(result);
}
result = new TestResultCollection(Name);
for (int i = 0; i < runners.Count; i++)
{
var runner = runners [i];
result.AddChild(await Run(category, runner, cancellationToken));
}
AddWarnings(result);
return(result);
}
void AddWarnings(TestResultCollection result)
{
result.AddWarnings(Warnings);
foreach (var test in tests)
{
result.AddWarnings(test.Warnings);
}
}
public async Task <TestResultCollection> Run(ITestCategory category, int repeatCount,
CancellationToken cancellationToken)
{
var result = new TestResultCollection(Name);
try {
for (int i = 0; i < hosts.Length; i++)
{
await hosts[i].SetUp(this);
}
} catch (Exception ex) {
Log("SetUp failed: {0}", ex);
result.AddChild(new TestError(Name, "SetUp failed", ex));
return(result);
}
if (repeatCount == 1)
{
CurrentIteration = MaxIterations = 1;
await DoRun(category, result, cancellationToken);
}
else
{
MaxIterations = repeatCount;
for (CurrentIteration = 0; CurrentIteration < repeatCount; CurrentIteration++)
{
var name = string.Format("{0} (iteration {1})", Name, CurrentIteration + 1);
var iteration = new TestResultCollection(name);
result.AddChild(iteration);
await DoRun(category, iteration, cancellationToken);
}
}
try {
for (int i = 0; i < hosts.Length; i++)
{
await hosts[i].TearDown(this);
}
} catch (Exception ex) {
Log("TearDown failed: {0}", ex);
result.AddChild(new TestError(Name, "TearDown failed", ex));
}
OnStatusMessageEvent("Test suite finished.");
return(result);
}
async Task DoRun(ITestCategory category, TestResultCollection result,
CancellationToken cancellationToken)
{
foreach (var fixture in fixtures)
{
if (!fixture.IsEnabled(category))
{
continue;
}
try {
result.AddChild(await fixture.Run(category, cancellationToken));
} catch (Exception ex) {
Log("Test fixture {0} failed: {1}", fixture.Name, ex);
result.AddChild(new TestError(fixture.Name, "Test fixture failed", ex));
}
}
}
public async Task <TestResult> Run(TestContext context, CancellationToken cancellationToken)
{
var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
if (Attribute.Timeout > 0)
{
cts.CancelAfter(Attribute.Timeout);
}
var oldConfig = context.Configuration;
try {
context.Configuration = Configuration;
var attr = Method.GetCustomAttribute <ExpectedExceptionAttribute> ();
TestResult result;
if (attr != null)
{
result = await ExpectingException(context, attr.ExpectedException, cts.Token);
}
else
{
result = await ExpectingSuccess(context, cts.Token);
}
if (!context.HasWarnings)
{
return(result);
}
var collection = new TestResultCollection(result.Name);
collection.AddChild(result);
collection.AddWarnings(context.Warnings);
return(collection);
} catch (Exception ex) {
Log("Test {0} failed: {1}", Name, ex);
return(new TestError(Name, null, ex));
} finally {
context.Configuration = oldConfig;
}
}
async Task <TestResultCollection> Run(ITestCategory category, ITestRunner runner,
CancellationToken cancellationToken)
{
string name = Name;
if (runner.Name != null)
{
name = string.Format("{0} ({1})", Name, runner.Name);
}
var result = new TestResultCollection(name);
var selected = Tests.Where(test => TestSuite.Filter(test, runner, category));
if (Attribute.ReuseContext)
{
await Run(runner, result, selected, cancellationToken);
return(result);
}
foreach (var test in selected)
{
var array = new TestCase[] { test };
bool reuseContext = GetReuseContext(test);
int repeat = reuseContext ? 1 : GetRepeat(test);
for (int i = 0; i < repeat; i++)
{
await Run(runner, result, array, cancellationToken);
}
}
return(result);
}
async Task Run(ITestRunner runner, TestResultCollection result,
IEnumerable <TestCase> selected, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
if (Attribute.Timeout > 0)
{
cts.CancelAfter(Attribute.Timeout);
}
var context = runner.CreateContext(this);
context.Configuration = Configuration;
try {
SetUp(context);
} catch (Exception ex) {
Log("{0}.SetUp failed: {1}", result.Name, ex);
result.AddChild(new TestError(result.Name, "SetUp failed", ex));
return;
}
foreach (var test in selected)
{
if (cts.Token.IsCancellationRequested)
{
break;
}
bool reuseContext = GetReuseContext(test);
int repeat = reuseContext ? GetRepeat(test) : 1;
string iteration;
if (Suite.MaxIterations == 1)
{
iteration = string.Empty;
}
else
{
iteration = string.Format(
" (iteration {0}/{1})", Suite.CurrentIteration + 1,
Suite.MaxIterations);
}
string name = Name;
if (runner.Name != null)
{
name = string.Format(
"{0}.{1} ({2}){3}", Name, test.Name, runner.Name, iteration);
}
else
{
name = string.Format(
"{0}.{1} ({2}", Name, test.Name, iteration);
}
for (int i = 0; i < repeat; i++)
{
string thisIter;
if (repeat == 1)
{
thisIter = string.Empty;
}
else
{
thisIter = string.Format(" (iteration {0}/{1})", i + 1, repeat);
}
Suite.OnStatusMessageEvent("Running {0}{1}", name, thisIter);
try {
var retval = await Run(test, context, cts.Token);
result.AddChild(retval);
} catch (Exception ex) {
Log("{0} failed: {1}", test.Name, ex);
result.AddChild(new TestError(result.Name, null, ex));
}
}
}
try {
TearDown(context);
} catch (Exception ex) {
Log("{0}.TearDown failed: {1}", result.Name, ex);
result.AddChild(new TestError(result.Name, "TearDown failed", ex));
}
cts.Token.ThrowIfCancellationRequested();
}
async Task Run (ITestRunner runner, TestResultCollection result,
IEnumerable<TestCase> selected, CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested ();
var cts = CancellationTokenSource.CreateLinkedTokenSource (cancellationToken);
if (Attribute.Timeout > 0)
cts.CancelAfter (Attribute.Timeout);
var context = runner.CreateContext (this);
context.Configuration = Configuration;
try {
SetUp (context);
} catch (Exception ex) {
Log ("{0}.SetUp failed: {1}", result.Name, ex);
result.AddChild (new TestError (result.Name, "SetUp failed", ex));
return;
}
foreach (var test in selected) {
if (cts.Token.IsCancellationRequested)
break;
bool reuseContext = GetReuseContext (test);
int repeat = reuseContext ? GetRepeat (test) : 1;
string iteration;
if (Suite.MaxIterations == 1)
iteration = string.Empty;
else
iteration = string.Format (
" (iteration {0}/{1})", Suite.CurrentIteration+1,
Suite.MaxIterations);
string name = Name;
if (runner.Name != null)
name = string.Format (
"{0}.{1} ({2}){3}", Name, test.Name, runner.Name, iteration);
else
name = string.Format (
"{0}.{1} ({2}", Name, test.Name, iteration);
for (int i = 0; i < repeat; i++) {
string thisIter;
if (repeat == 1)
thisIter = string.Empty;
else
thisIter = string.Format (" (iteration {0}/{1})", i+1, repeat);
Suite.OnStatusMessageEvent ("Running {0}{1}", name, thisIter);
try {
var retval = await Run (test, context, cts.Token);
result.AddChild (retval);
} catch (Exception ex) {
Log ("{0} failed: {1}", test.Name, ex);
result.AddChild (new TestError (result.Name, null, ex));
}
}
}
try {
TearDown (context);
} catch (Exception ex) {
Log ("{0}.TearDown failed: {1}", result.Name, ex);
result.AddChild (new TestError (result.Name, "TearDown failed", ex));
}
cts.Token.ThrowIfCancellationRequested ();
}
async Task<TestResultCollection> Run (ITestCategory category, ITestRunner runner,
CancellationToken cancellationToken)
{
string name = Name;
if (runner.Name != null)
name = string.Format ("{0} ({1})", Name, runner.Name);
var result = new TestResultCollection (name);
var selected = Tests.Where (test => TestSuite.Filter (test, runner, category));
if (Attribute.ReuseContext) {
await Run (runner, result, selected, cancellationToken);
return result;
}
foreach (var test in selected) {
var array = new TestCase[] { test };
bool reuseContext = GetReuseContext (test);
int repeat = reuseContext ? 1 : GetRepeat (test);
for (int i = 0; i < repeat; i++)
await Run (runner, result, array, cancellationToken);
}
return result;
}
public async Task<TestResultCollection> Run (ITestCategory category,
CancellationToken cancellationToken)
{
TestResultCollection result;
var runners = GetTestRunners (category);
if (runners.Count == 1) {
result = await Run (category, runners [0], cancellationToken);
AddWarnings (result);
return result;
}
result = new TestResultCollection (Name);
for (int i = 0; i < runners.Count; i++) {
var runner = runners [i];
result.AddChild (await Run (category, runner, cancellationToken));
}
AddWarnings (result);
return result;
}
void AddWarnings (TestResultCollection result)
{
result.AddWarnings (Warnings);
foreach (var test in tests) {
result.AddWarnings (test.Warnings);
}
}
public void SetResult (TestResultCollection results)
{
root = new ResultWrapper (results);
OnChangedEvent ();
}
public abstract void Visit (TestResultCollection node);
public abstract void Visit(TestResultCollection node);
public async Task<TestResult> Run (TestContext context, CancellationToken cancellationToken)
{
var cts = CancellationTokenSource.CreateLinkedTokenSource (cancellationToken);
if (Attribute.Timeout > 0)
cts.CancelAfter (Attribute.Timeout);
var oldConfig = context.Configuration;
try {
context.Configuration = Configuration;
var attr = Method.GetCustomAttribute<ExpectedExceptionAttribute> ();
TestResult result;
if (attr != null)
result = await ExpectingException (context, attr.ExceptionType, cts.Token);
else
result = await ExpectingSuccess (context, cts.Token);
if (!context.HasWarnings)
return result;
var collection = new TestResultCollection (result.Name);
collection.AddChild (result);
collection.AddWarnings (context.Warnings);
return collection;
} catch (Exception ex) {
Log ("Test {0} failed: {1}", Name, ex);
return new TestError (Name, null, ex);
} finally {
context.Configuration = oldConfig;
}
}