public static async ValueTask FailureInBeforeTestAssemblyFinished_ReportsCleanupFailure_DoesNotIncludeExceptionsFromAfterTestAssemblyStarting()
{
var thisAssembly = Assembly.GetExecutingAssembly();
var messages = new List <_MessageSinkMessage>();
var messageSink = SpyMessageSink.Create(messages: messages);
var testCases = new[] { TestCaseForTestCollection() };
await using var runner = TestableTestAssemblyRunner.Create(messageSink, testCases: testCases);
var startingException = new DivideByZeroException();
var finishedException = new InvalidOperationException();
runner.AfterTestAssemblyStarting_Callback = aggregator => aggregator.Add(startingException);
runner.BeforeTestAssemblyFinished_Callback = aggregator => aggregator.Add(finishedException);
await runner.RunAsync();
var assemblyStarting = Assert.Single(messages.OfType <_TestAssemblyStarting>());
var cleanupFailure = Assert.Single(messages.OfType <_TestAssemblyCleanupFailure>());
#if NETFRAMEWORK
Assert.Equal(thisAssembly.GetLocalCodeBase(), assemblyStarting.AssemblyPath);
Assert.Equal(runner.TestAssembly.ConfigFileName, assemblyStarting.ConfigFilePath);
#endif
Assert.Equal(typeof(InvalidOperationException).FullName, cleanupFailure.ExceptionTypes.Single());
}
public static async ValueTask OrdererUsedToOrderTestCollections()
{
var collection1 = Mocks.TestCollection(displayName: "AAA", uniqueID: "collection-1");
var testCase1a = TestCaseForTestCollection(collection1);
var testCase1b = TestCaseForTestCollection(collection1);
var collection2 = Mocks.TestCollection(displayName: "ZZZZ", uniqueID: "collection-2");
var testCase2a = TestCaseForTestCollection(collection2);
var testCase2b = TestCaseForTestCollection(collection2);
var collection3 = Mocks.TestCollection(displayName: "MM", uniqueID: "collection-3");
var testCase3a = TestCaseForTestCollection(collection3);
var testCase3b = TestCaseForTestCollection(collection3);
var testCases = new[] { testCase1a, testCase3a, testCase2a, testCase3b, testCase2b, testCase1b };
var runner = TestableTestAssemblyRunner.Create(testCases: testCases, testCollectionOrderer: new DescendingDisplayNameCollectionOrderer());
await runner.RunAsync();
Assert.Collection(
runner.CollectionsRun,
collection =>
{
Assert.Same(collection2, collection.Item1);
Assert.Equal(new[] { testCase2a, testCase2b }, collection.Item2);
},
collection =>
{
Assert.Same(collection3, collection.Item1);
Assert.Equal(new[] { testCase3a, testCase3b }, collection.Item2);
},
collection =>
{
Assert.Same(collection1, collection.Item1);
Assert.Equal(new[] { testCase1a, testCase1b }, collection.Item2);
}
);
}
public static async void FailureInBeforeTestAssemblyFinished_ReportsCleanupFailure_DoesNotIncludeExceptionsFromAfterTestAssemblyStarting()
{
var thisAssembly = Assembly.GetExecutingAssembly();
var thisAppDomain = AppDomain.CurrentDomain;
var messages = new List <IMessageSinkMessage>();
var messageSink = SpyMessageSink.Create(messages: messages);
var testCases = new[] { Mocks.TestCase() };
var runner = TestableTestAssemblyRunner.Create(messageSink, testCases: testCases);
var startingException = new DivideByZeroException();
var finishedException = new InvalidOperationException();
runner.AfterTestAssemblyStarting_Callback = aggregator => aggregator.Add(startingException);
runner.BeforeTestAssemblyFinished_Callback = aggregator => aggregator.Add(finishedException);
await runner.RunAsync();
var cleanupFailure = Assert.Single(messages.OfType <ITestAssemblyCleanupFailure>());
#if NETFRAMEWORK
Assert.Equal(thisAssembly.GetLocalCodeBase(), cleanupFailure.TestAssembly.Assembly.AssemblyPath);
Assert.Equal(thisAppDomain.SetupInformation.ConfigurationFile, cleanupFailure.TestAssembly.ConfigFileName);
#endif
Assert.Equal(testCases, cleanupFailure.TestCases);
Assert.Equal(typeof(InvalidOperationException).FullName, cleanupFailure.ExceptionTypes.Single());
}
public static async void FailureInQueueOfTestAssemblyStarting_DoesNotQueueTestAssemblyFinished_DoesNotRunTestCollections()
{
var messages = new List <IMessageSinkMessage>();
var messageSink = Substitute.For <IMessageSink>();
messageSink.OnMessage(null)
.ReturnsForAnyArgs(callInfo =>
{
var msg = callInfo.Arg <IMessageSinkMessage>();
messages.Add(msg);
if (msg is ITestAssemblyStarting)
{
throw new InvalidOperationException();
}
return(true);
});
var runner = TestableTestAssemblyRunner.Create(messageSink);
await Assert.ThrowsAsync <InvalidOperationException>(() => runner.RunAsync());
var starting = Assert.Single(messages);
Assert.IsAssignableFrom <ITestAssemblyStarting>(starting);
Assert.Empty(runner.CollectionsRun);
}
public static async void TestsAreGroupedByCollection()
{
var collection1 = Mocks.TestCollection(displayName: "1");
var testCase1a = Mocks.TestCase(collection1);
var testCase1b = Mocks.TestCase(collection1);
var collection2 = Mocks.TestCollection(displayName: "2");
var testCase2a = Mocks.TestCase(collection2);
var testCase2b = Mocks.TestCase(collection2);
var runner = TestableTestAssemblyRunner.Create(testCases: new[] { testCase1a, testCase2a, testCase2b, testCase1b });
await runner.RunAsync();
Assert.Collection(runner.CollectionsRun.OrderBy(c => c.Item1.DisplayName),
tuple =>
{
Assert.Same(collection1, tuple.Item1);
Assert.Collection(tuple.Item2,
testCase => Assert.Same(testCase1a, testCase),
testCase => Assert.Same(testCase1b, testCase)
);
},
tuple =>
{
Assert.Same(collection2, tuple.Item1);
Assert.Collection(tuple.Item2,
testCase => Assert.Same(testCase2a, testCase),
testCase => Assert.Same(testCase2b, testCase)
);
}
);
}
public static async void TestCaseOrdererWhichThrowsLogsMessageAndDoesNotReorderTests()
{
Thread.CurrentThread.CurrentUICulture = CultureInfo.InvariantCulture;
var collection1 = Mocks.TestCollection(displayName: "AAA");
var testCase1 = Mocks.TestCase(collection1);
var collection2 = Mocks.TestCollection(displayName: "ZZZZ");
var testCase2 = Mocks.TestCase(collection2);
var collection3 = Mocks.TestCollection(displayName: "MM");
var testCase3 = Mocks.TestCase(collection3);
var testCases = new[] { testCase1, testCase2, testCase3 };
var runner = TestableTestAssemblyRunner.Create(testCases: testCases);
runner.TestCollectionOrderer = new ThrowingOrderer();
await runner.RunAsync();
Assert.Collection(runner.CollectionsRun,
collection => Assert.Same(collection1, collection.Item1),
collection => Assert.Same(collection2, collection.Item1),
collection => Assert.Same(collection3, collection.Item1)
);
var diagnosticMessage = Assert.Single(runner.DiagnosticMessages.Cast <IDiagnosticMessage>());
Assert.StartsWith("Test collection orderer 'TestAssemblyRunnerTests+TestCollectionOrderer+ThrowingOrderer' threw 'System.DivideByZeroException' during ordering: Attempted to divide by zero.", diagnosticMessage.Message);
}
public static async void TestContextInspection()
{
var runner = TestableTestAssemblyRunner.Create();
await runner.RunAsync();
Assert.NotNull(runner.AfterTestAssemblyStarting_Context);
Assert.Equal(TestEngineStatus.Initializing, runner.AfterTestAssemblyStarting_Context.TestAssemblyStatus);
Assert.Equal(TestPipelineStage.TestAssemblyExecution, runner.AfterTestAssemblyStarting_Context.PipelineStage);
Assert.Null(runner.AfterTestAssemblyStarting_Context.TestCollectionStatus);
Assert.Null(runner.AfterTestAssemblyStarting_Context.TestClassStatus);
Assert.Null(runner.AfterTestAssemblyStarting_Context.TestMethodStatus);
Assert.Null(runner.AfterTestAssemblyStarting_Context.TestCaseStatus);
Assert.Null(runner.AfterTestAssemblyStarting_Context.TestStatus);
Assert.Same(runner.TestAssembly, runner.AfterTestAssemblyStarting_Context.TestAssembly);
Assert.NotNull(runner.RunTestCollectionAsync_Context);
Assert.Equal(TestEngineStatus.Running, runner.RunTestCollectionAsync_Context.TestAssemblyStatus);
Assert.Null(runner.RunTestCollectionAsync_Context.TestCollectionStatus);
Assert.Null(runner.RunTestCollectionAsync_Context.TestClassStatus);
Assert.Null(runner.RunTestCollectionAsync_Context.TestMethodStatus);
Assert.Null(runner.RunTestCollectionAsync_Context.TestCaseStatus);
Assert.Null(runner.RunTestCollectionAsync_Context.TestStatus);
Assert.Same(runner.TestAssembly, runner.RunTestCollectionAsync_Context.TestAssembly);
Assert.NotNull(runner.BeforeTestAssemblyFinished_Context);
Assert.Equal(TestEngineStatus.CleaningUp, runner.BeforeTestAssemblyFinished_Context.TestAssemblyStatus);
Assert.Null(runner.BeforeTestAssemblyFinished_Context.TestCollectionStatus);
Assert.Null(runner.BeforeTestAssemblyFinished_Context.TestClassStatus);
Assert.Null(runner.BeforeTestAssemblyFinished_Context.TestMethodStatus);
Assert.Null(runner.BeforeTestAssemblyFinished_Context.TestCaseStatus);
Assert.Null(runner.BeforeTestAssemblyFinished_Context.TestStatus);
Assert.Same(runner.TestAssembly, runner.BeforeTestAssemblyFinished_Context.TestAssembly);
}
public static async ValueTask TestCaseOrdererWhichThrowsLogsMessageAndDoesNotReorderTests()
{
var collection1 = Mocks.TestCollection(displayName: "AAA", uniqueID: "collection-1");
var testCase1 = TestCaseForTestCollection(collection1);
var collection2 = Mocks.TestCollection(displayName: "ZZZZ", uniqueID: "collection-2");
var testCase2 = TestCaseForTestCollection(collection2);
var collection3 = Mocks.TestCollection(displayName: "MM", uniqueID: "collection-3");
var testCase3 = TestCaseForTestCollection(collection3);
var testCases = new[] { testCase1, testCase2, testCase3 };
await using var runner = TestableTestAssemblyRunner.Create(testCases: testCases);
runner.TestCollectionOrderer = new ThrowingOrderer();
await runner.RunAsync();
Assert.Collection(
runner.CollectionsRun,
collection => Assert.Same(collection1, collection.Item1),
collection => Assert.Same(collection2, collection.Item1),
collection => Assert.Same(collection3, collection.Item1)
);
var diagnosticMessage = Assert.Single(runner.DiagnosticMessages.Cast <_DiagnosticMessage>());
Assert.StartsWith("Test collection orderer 'TestAssemblyRunnerTests+TestCollectionOrderer+ThrowingOrderer' threw 'System.DivideByZeroException' during ordering: Attempted to divide by zero.", diagnosticMessage.Message);
}
public static void DefaultMessageBus()
{
var runner = TestableTestAssemblyRunner.Create();
var messageBus = runner.CreateMessageBus_Public();
Assert.IsType <MessageBus>(messageBus);
}
public static async ValueTask DefaultMessageBus()
{
await using var runner = TestableTestAssemblyRunner.Create();
using var messageBus = runner.CreateMessageBus_Public();
Assert.IsType <MessageBus>(messageBus);
}
public static void SyncMessageBusOption()
{
var runner = TestableTestAssemblyRunner.Create(options: new XunitExecutionOptions {
SynchronousMessageReporting = true
});
var messageBus = runner.CreateMessageBus_Public();
Assert.IsType <SynchronousMessageBus>(messageBus);
}
public static async ValueTask Cancellation_TestAssemblyStarting_DoesNotCallExtensibilityCallbacks()
{
var messageSink = SpyMessageSink.Create(msg => !(msg is _TestAssemblyStarting));
var runner = TestableTestAssemblyRunner.Create(messageSink);
await runner.RunAsync();
Assert.False(runner.AfterTestAssemblyStarting_Called);
Assert.False(runner.BeforeTestAssemblyFinished_Called);
}
public static async void Cancellation_TestAssemblyFinished_CallsCallExtensibilityCallbacks()
{
var messageSink = SpyMessageSink.Create(msg => !(msg is ITestAssemblyFinished));
var runner = TestableTestAssemblyRunner.Create(messageSink);
await runner.RunAsync();
Assert.True(runner.AfterTestAssemblyStarting_Called);
Assert.True(runner.BeforeTestAssemblyFinished_Called);
}
public static void SyncMessageBusOption()
{
var executionOptions = TestFrameworkOptions.ForExecution();
executionOptions.SetSynchronousMessageReporting(true);
var runner = TestableTestAssemblyRunner.Create(executionOptions: executionOptions);
var messageBus = runner.CreateMessageBus_Public();
Assert.IsType <SynchronousMessageBus>(messageBus);
}
public static async void SignalingCancellationStopsRunningCollections()
{
var collection1 = Mocks.TestCollection();
var testCase1 = Mocks.TestCase(collection1);
var collection2 = Mocks.TestCollection();
var testCase2 = Mocks.TestCase(collection2);
var runner = TestableTestAssemblyRunner.Create(testCases: new[] { testCase1, testCase2 }, cancelInRunTestCollectionAsync: true);
await runner.RunAsync();
Assert.Single(runner.CollectionsRun);
}
public static async void FailureInAfterTestAssemblyStarting_GivesErroredAggregatorToTestCollectionRunner_NoCleanupFailureMessage()
{
var messages = new List <IMessageSinkMessage>();
var messageSink = SpyMessageSink.Create(messages: messages);
var runner = TestableTestAssemblyRunner.Create(messageSink);
var ex = new DivideByZeroException();
runner.AfterTestAssemblyStarting_Callback = aggregator => aggregator.Add(ex);
await runner.RunAsync();
Assert.Same(ex, runner.RunTestCollectionAsync_AggregatorResult);
Assert.Empty(messages.OfType <ITestAssemblyCleanupFailure>());
}
public static async ValueTask Messages()
{
var summary = new RunSummary {
Total = 4, Failed = 2, Skipped = 1, Time = 21.12m
};
var messages = new List <_MessageSinkMessage>();
var messageSink = SpyMessageSink.Create(messages: messages);
await using var runner = TestableTestAssemblyRunner.Create(messageSink, summary);
var thisAssembly = Assembly.GetExecutingAssembly();
var result = await runner.RunAsync();
Assert.Equal(4, result.Total);
Assert.Equal(2, result.Failed);
Assert.Equal(1, result.Skipped);
Assert.NotEqual(21.12m, result.Time); // Uses clock time, not result time
Assert.Collection(
messages,
msg =>
{
var starting = Assert.IsAssignableFrom <_TestAssemblyStarting>(msg);
#if NETFRAMEWORK
Assert.Equal(thisAssembly.GetLocalCodeBase(), starting.AssemblyPath);
Assert.Equal(runner.TestAssembly.ConfigFileName, starting.ConfigFilePath);
Assert.Equal(".NETFramework,Version=v4.7.2", starting.TargetFramework);
#else
Assert.Equal(".NETCoreApp,Version=v2.1", starting.TargetFramework);
#endif
Assert.InRange(starting.StartTime, DateTime.Now.AddMinutes(-15), DateTime.Now);
Assert.Equal("The test framework environment", starting.TestEnvironment);
Assert.Equal("The test framework display name", starting.TestFrameworkDisplayName);
Assert.Equal("assembly-id", starting.AssemblyUniqueID);
},
msg =>
{
var finished = Assert.IsAssignableFrom <_TestAssemblyFinished>(msg);
Assert.Equal("assembly-id", finished.AssemblyUniqueID);
Assert.Equal(result.Time, finished.ExecutionTime);
Assert.Equal(2, finished.TestsFailed);
Assert.Equal(4, finished.TestsRun);
Assert.Equal(1, finished.TestsSkipped);
}
);
}
public static async void Messages()
{
var summary = new RunSummary {
Total = 4, Failed = 2, Skipped = 1, Time = 21.12m
};
var messages = new List <IMessageSinkMessage>();
var messageSink = SpyMessageSink.Create(messages: messages);
var runner = TestableTestAssemblyRunner.Create(messageSink, summary);
var thisAssembly = Assembly.GetExecutingAssembly();
var thisAppDomain = AppDomain.CurrentDomain;
var result = await runner.RunAsync();
Assert.Equal(4, result.Total);
Assert.Equal(2, result.Failed);
Assert.Equal(1, result.Skipped);
Assert.NotEqual(21.12m, result.Time); // Uses clock time, not result time
Assert.Collection(messages,
msg =>
{
var starting = Assert.IsAssignableFrom <ITestAssemblyStarting>(msg);
#if NETFRAMEWORK
Assert.Equal(thisAssembly.GetLocalCodeBase(), starting.TestAssembly.Assembly.AssemblyPath);
Assert.Equal(thisAppDomain.SetupInformation.ConfigurationFile, starting.TestAssembly.ConfigFileName);
#endif
Assert.InRange(starting.StartTime, DateTime.Now.AddMinutes(-15), DateTime.Now);
Assert.Equal("The test framework environment", starting.TestEnvironment);
Assert.Equal("The test framework display name", starting.TestFrameworkDisplayName);
},
msg =>
{
var finished = Assert.IsAssignableFrom <ITestAssemblyFinished>(msg);
Assert.Equal(4, finished.TestsRun);
Assert.Equal(2, finished.TestsFailed);
Assert.Equal(1, finished.TestsSkipped);
Assert.Equal(result.Time, finished.ExecutionTime);
}
);
}
public static async void OrdererUsedToOrderTestCases()
{
var collection1 = Mocks.TestCollection(displayName: "AAA");
var testCase1a = Mocks.TestCase(collection1);
var testCase1b = Mocks.TestCase(collection1);
var collection2 = Mocks.TestCollection(displayName: "ZZZZ");
var testCase2a = Mocks.TestCase(collection2);
var testCase2b = Mocks.TestCase(collection2);
var collection3 = Mocks.TestCollection(displayName: "MM");
var testCase3a = Mocks.TestCase(collection3);
var testCase3b = Mocks.TestCase(collection3);
var testCases = new[] { testCase1a, testCase3a, testCase2a, testCase3b, testCase2b, testCase1b };
var runner = TestableTestAssemblyRunner.Create(testCases: testCases);
runner.TestCollectionOrderer = new MyTestCollectionOrderer();
await runner.RunAsync();
Assert.Collection(runner.CollectionsRun,
collection =>
{
Assert.Same(collection2, collection.Item1);
Assert.Equal(new[] { testCase2a, testCase2b }, collection.Item2);
},
collection =>
{
Assert.Same(collection3, collection.Item1);
Assert.Equal(new[] { testCase3a, testCase3b }, collection.Item2);
},
collection =>
{
Assert.Same(collection1, collection.Item1);
Assert.Equal(new[] { testCase1a, testCase1b }, collection.Item2);
}
);
}
public static void DefaultTestCaseOrderer()
{
var runner = TestableTestAssemblyRunner.Create();
Assert.IsType <DefaultTestCollectionOrderer>(runner.TestCollectionOrderer);
}
public static async ValueTask DefaultTestCaseOrderer()
{
await using var runner = TestableTestAssemblyRunner.Create();
Assert.IsType <DefaultTestCollectionOrderer>(runner.TestCollectionOrderer);
}
