public static async void Cancellation_TestAssemblyStarting_DoesNotCallExtensibilityCallbacks()
{
var messageSink = SpyMessageSink.Create(msg => !(msg is ITestAssemblyStarting));
var runner = TestableTestAssemblyRunner.Create(messageSink);
await runner.RunAsync();
Assert.False(runner.AfterTestAssemblyStarting_Called);
Assert.False(runner.BeforeTestAssemblyFinished_Called);
}
public static async ValueTask Cancellation_TestAssemblyFinished_CallsCallExtensibilityCallbacks()
{
var messageSink = SpyMessageSink.Create(msg => !(msg is _TestAssemblyFinished));
var runner = TestableTestAssemblyRunner.Create(messageSink);
await runner.RunAsync();
Assert.True(runner.AfterTestAssemblyStarting_Called);
Assert.True(runner.BeforeTestAssemblyFinished_Called);
}
public static void SyncMessageBusOption()
{
var runner = TestableTestAssemblyRunner.Create(options: new XunitExecutionOptions {
SynchronousMessageReporting = true
});
var messageBus = runner.CreateMessageBus_Public();
Assert.IsType <SynchronousMessageBus>(messageBus);
}
public static void SyncMessageBusOption()
{
var executionOptions = TestFrameworkOptions.ForExecution();
executionOptions.SetSynchronousMessageReporting(true);
var runner = TestableTestAssemblyRunner.Create(executionOptions: executionOptions);
var messageBus = runner.CreateMessageBus_Public();
Assert.IsType <SynchronousMessageBus>(messageBus);
}
public static async void SignalingCancellationStopsRunningCollections()
{
var collection1 = Mocks.TestCollection();
var testCase1 = Mocks.TestCase(collection1);
var collection2 = Mocks.TestCollection();
var testCase2 = Mocks.TestCase(collection2);
var runner = TestableTestAssemblyRunner.Create(testCases: new[] { testCase1, testCase2 }, cancelInRunTestCollectionAsync: true);
await runner.RunAsync();
Assert.Single(runner.CollectionsRun);
}
public static async void FailureInAfterTestAssemblyStarting_GivesErroredAggregatorToTestCollectionRunner_NoCleanupFailureMessage()
{
var messages = new List <IMessageSinkMessage>();
var messageSink = SpyMessageSink.Create(messages: messages);
var runner = TestableTestAssemblyRunner.Create(messageSink);
var ex = new DivideByZeroException();
runner.AfterTestAssemblyStarting_Callback = aggregator => aggregator.Add(ex);
await runner.RunAsync();
Assert.Same(ex, runner.RunTestCollectionAsync_AggregatorResult);
Assert.Empty(messages.OfType <ITestAssemblyCleanupFailure>());
}
public static async ValueTask Messages()
{
var summary = new RunSummary {
Total = 4, Failed = 2, Skipped = 1, Time = 21.12m
};
var messages = new List <_MessageSinkMessage>();
var messageSink = SpyMessageSink.Create(messages: messages);
await using var runner = TestableTestAssemblyRunner.Create(messageSink, summary);
var thisAssembly = Assembly.GetExecutingAssembly();
var result = await runner.RunAsync();
Assert.Equal(4, result.Total);
Assert.Equal(2, result.Failed);
Assert.Equal(1, result.Skipped);
Assert.NotEqual(21.12m, result.Time); // Uses clock time, not result time
Assert.Collection(
messages,
msg =>
{
var starting = Assert.IsAssignableFrom <_TestAssemblyStarting>(msg);
#if NETFRAMEWORK
Assert.Equal(thisAssembly.GetLocalCodeBase(), starting.AssemblyPath);
Assert.Equal(runner.TestAssembly.ConfigFileName, starting.ConfigFilePath);
Assert.Equal(".NETFramework,Version=v4.7.2", starting.TargetFramework);
#else
Assert.Equal(".NETCoreApp,Version=v2.1", starting.TargetFramework);
#endif
Assert.InRange(starting.StartTime, DateTime.Now.AddMinutes(-15), DateTime.Now);
Assert.Equal("The test framework environment", starting.TestEnvironment);
Assert.Equal("The test framework display name", starting.TestFrameworkDisplayName);
Assert.Equal("assembly-id", starting.AssemblyUniqueID);
},
msg =>
{
var finished = Assert.IsAssignableFrom <_TestAssemblyFinished>(msg);
Assert.Equal("assembly-id", finished.AssemblyUniqueID);
Assert.Equal(result.Time, finished.ExecutionTime);
Assert.Equal(2, finished.TestsFailed);
Assert.Equal(4, finished.TestsRun);
Assert.Equal(1, finished.TestsSkipped);
}
);
}
public static async void Messages()
{
var summary = new RunSummary {
Total = 4, Failed = 2, Skipped = 1, Time = 21.12m
};
var messages = new List <IMessageSinkMessage>();
var messageSink = SpyMessageSink.Create(messages: messages);
var runner = TestableTestAssemblyRunner.Create(messageSink, summary);
var thisAssembly = Assembly.GetExecutingAssembly();
var thisAppDomain = AppDomain.CurrentDomain;
var result = await runner.RunAsync();
Assert.Equal(4, result.Total);
Assert.Equal(2, result.Failed);
Assert.Equal(1, result.Skipped);
Assert.NotEqual(21.12m, result.Time); // Uses clock time, not result time
Assert.Collection(messages,
msg =>
{
var starting = Assert.IsAssignableFrom <ITestAssemblyStarting>(msg);
#if NETFRAMEWORK
Assert.Equal(thisAssembly.GetLocalCodeBase(), starting.TestAssembly.Assembly.AssemblyPath);
Assert.Equal(thisAppDomain.SetupInformation.ConfigurationFile, starting.TestAssembly.ConfigFileName);
#endif
Assert.InRange(starting.StartTime, DateTime.Now.AddMinutes(-15), DateTime.Now);
Assert.Equal("The test framework environment", starting.TestEnvironment);
Assert.Equal("The test framework display name", starting.TestFrameworkDisplayName);
},
msg =>
{
var finished = Assert.IsAssignableFrom <ITestAssemblyFinished>(msg);
Assert.Equal(4, finished.TestsRun);
Assert.Equal(2, finished.TestsFailed);
Assert.Equal(1, finished.TestsSkipped);
Assert.Equal(result.Time, finished.ExecutionTime);
}
);
}
public static async void FailureInBeforeTestAssemblyFinished_ReportsCleanupFailure_DoesNotIncludeExceptionsFromAfterTestAssemblyStarting()
{
var thisAssembly = Assembly.GetExecutingAssembly();
var thisAppDomain = AppDomain.CurrentDomain;
var messages = new List <IMessageSinkMessage>();
var messageSink = SpyMessageSink.Create(messages: messages);
var testCases = new[] { Mocks.TestCase() };
var runner = TestableTestAssemblyRunner.Create(messageSink, testCases: testCases);
var startingException = new DivideByZeroException();
var finishedException = new InvalidOperationException();
runner.AfterTestAssemblyStarting_Callback = aggregator => aggregator.Add(startingException);
runner.BeforeTestAssemblyFinished_Callback = aggregator => aggregator.Add(finishedException);
await runner.RunAsync();
var cleanupFailure = Assert.Single(messages.OfType <ITestAssemblyCleanupFailure>());
Assert.Equal(thisAssembly.GetLocalCodeBase(), cleanupFailure.TestAssembly.Assembly.AssemblyPath);
Assert.Equal(thisAppDomain.SetupInformation.ConfigurationFile, cleanupFailure.TestAssembly.ConfigFileName);
Assert.Equal(testCases, cleanupFailure.TestCases);
Assert.Equal(typeof(InvalidOperationException).FullName, cleanupFailure.ExceptionTypes.Single());
}
public static async ValueTask OrdererUsedToOrderTestCases()
{
var collection1 = Mocks.TestCollection(displayName: "AAA", uniqueID: "collection-1");
var testCase1a = TestCaseForTestCollection(collection1);
var testCase1b = TestCaseForTestCollection(collection1);
var collection2 = Mocks.TestCollection(displayName: "ZZZZ", uniqueID: "collection-2");
var testCase2a = TestCaseForTestCollection(collection2);
var testCase2b = TestCaseForTestCollection(collection2);
var collection3 = Mocks.TestCollection(displayName: "MM", uniqueID: "collection-3");
var testCase3a = TestCaseForTestCollection(collection3);
var testCase3b = TestCaseForTestCollection(collection3);
var testCases = new[] { testCase1a, testCase3a, testCase2a, testCase3b, testCase2b, testCase1b };
await using var runner = TestableTestAssemblyRunner.Create(testCases: testCases);
runner.TestCollectionOrderer = new MyTestCollectionOrderer();
await runner.RunAsync();
Assert.Collection(
runner.CollectionsRun,
collection =>
{
Assert.Same(collection2, collection.Item1);
Assert.Equal(new[] { testCase2a, testCase2b }, collection.Item2);
},
collection =>
{
Assert.Same(collection3, collection.Item1);
Assert.Equal(new[] { testCase3a, testCase3b }, collection.Item2);
},
collection =>
{
Assert.Same(collection1, collection.Item1);
Assert.Equal(new[] { testCase1a, testCase1b }, collection.Item2);
}
);
}
public static async void TestCaseOrdererWhichThrowsLogsMessageAndDoesNotReorderTests()
{
var collection1 = Mocks.TestCollection(displayName: "AAA");
var testCase1 = Mocks.TestCase(collection1);
var collection2 = Mocks.TestCollection(displayName: "ZZZZ");
var testCase2 = Mocks.TestCase(collection2);
var collection3 = Mocks.TestCollection(displayName: "MM");
var testCase3 = Mocks.TestCase(collection3);
var testCases = new[] { testCase1, testCase2, testCase3 };
var runner = TestableTestAssemblyRunner.Create(testCases: testCases);
runner.TestCollectionOrderer = new ThrowingOrderer();
await runner.RunAsync();
Assert.Collection(runner.CollectionsRun,
collection => Assert.Same(collection1, collection.Item1),
collection => Assert.Same(collection2, collection.Item1),
collection => Assert.Same(collection3, collection.Item1)
);
var diagnosticMessage = Assert.Single(runner.DiagnosticMessages.Cast <IDiagnosticMessage>());
Assert.StartsWith("Test collection orderer 'TestAssemblyRunnerTests+TestCollectionOrderer+ThrowingOrderer' threw 'System.DivideByZeroException' during ordering: Attempted to divide by zero.", diagnosticMessage.Message);
}
public static async ValueTask FailureInBeforeTestAssemblyFinished_ReportsCleanupFailure_DoesNotIncludeExceptionsFromAfterTestAssemblyStarting()
{
var thisAssembly = Assembly.GetExecutingAssembly();
var messages = new List <_MessageSinkMessage>();
var messageSink = SpyMessageSink.Create(messages: messages);
var testCases = new[] { TestCaseForTestCollection() };
var runner = TestableTestAssemblyRunner.Create(messageSink, testCases: testCases);
var startingException = new DivideByZeroException();
var finishedException = new InvalidOperationException();
runner.AfterTestAssemblyStarting_Callback = aggregator => aggregator.Add(startingException);
runner.BeforeTestAssemblyFinished_Callback = aggregator => aggregator.Add(finishedException);
await runner.RunAsync();
var assemblyStarting = Assert.Single(messages.OfType <_TestAssemblyStarting>());
var cleanupFailure = Assert.Single(messages.OfType <_TestAssemblyCleanupFailure>());
#if NETFRAMEWORK
Assert.Equal(thisAssembly.GetLocalCodeBase(), assemblyStarting.AssemblyPath);
Assert.Equal(runner.TestAssembly.ConfigFileName, assemblyStarting.ConfigFilePath);
#endif
Assert.Equal(typeof(InvalidOperationException).FullName, cleanupFailure.ExceptionTypes.Single());
}
public static void DefaultTestCaseOrderer()
{
var runner = TestableTestAssemblyRunner.Create();
Assert.IsType <DefaultTestCollectionOrderer>(runner.TestCollectionOrderer);
}
public static async ValueTask DefaultTestCaseOrderer()
{
await using var runner = TestableTestAssemblyRunner.Create();
Assert.IsType <DefaultTestCollectionOrderer>(runner.TestCollectionOrderer);
}
