protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator)
{
// copy from XunitTestRunner
var output = string.Empty;
TestOutputHelper testOutputHelper = null;
foreach (object obj in ConstructorArguments)
{
testOutputHelper = obj as TestOutputHelper;
if (testOutputHelper != null)
{
break;
}
}
if (testOutputHelper != null)
{
testOutputHelper.Initialize(MessageBus, Test);
}
var executionTime = await InvokeTestMethodAsync(aggregator);
if (testOutputHelper != null)
{
output = testOutputHelper.Output;
testOutputHelper.Uninitialize();
}
// save the exceptions that were thrown during the test, in case this is needed
ExceptionAggregator?.Aggregate(aggregator);
return(Tuple.Create(executionTime, output));
}
/// <inheritdoc />
protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator)
{
var output = string.Empty;
TestOutputHelper testOutputHelper = null;
foreach (var obj in ConstructorArguments)
{
testOutputHelper = obj as TestOutputHelper;
if (testOutputHelper != null)
{
break;
}
}
testOutputHelper?.Initialize(MessageBus, Test);
//TODO: Add execution time, if strictly necessary - the NBench values are certainly more accurate.
var executionTime = await InvokeTestMethodAsync(aggregator);
if (testOutputHelper != null)
{
output = testOutputHelper.Output;
testOutputHelper.Uninitialize();
}
return(Tuple.Create(executionTime, output));
}
public void WriteLine(string outputText, string expected)
{
var output = new TestOutputHelper();
var messageBus = new SpyMessageBus();
var test = Substitute.For <_ITest>();
test.UniqueID.Returns("test-id");
test.TestCase.UniqueID.Returns("case-id");
test.TestCase.TestMethod.UniqueID.Returns("method-id");
test.TestCase.TestMethod.TestClass.UniqueID.Returns("class-id");
test.TestCase.TestMethod.TestClass.TestCollection.UniqueID.Returns("coll-id");
test.TestCase.TestMethod.TestClass.TestCollection.TestAssembly.UniqueID.Returns("asm-id");
output.Initialize(messageBus, test);
output.WriteLine(outputText);
var message = Assert.Single(messageBus.Messages);
var outputMessage = Assert.IsType <_TestOutput>(message);
Assert.Equal("asm-id", outputMessage.AssemblyUniqueID);
Assert.Equal(expected + Environment.NewLine, outputMessage.Output);
Assert.Equal("case-id", outputMessage.TestCaseUniqueID);
Assert.Equal("class-id", outputMessage.TestClassUniqueID);
Assert.Equal("coll-id", outputMessage.TestCollectionUniqueID);
Assert.Equal("method-id", outputMessage.TestMethodUniqueID);
Assert.Equal("test-id", outputMessage.TestUniqueID);
Assert.Equal(expected + Environment.NewLine, output.Output);
}
protected override async Task <decimal> InvokeTestMethodAsync(object testClassInstance)
{
var output = new TestOutputHelper();
output.Initialize(MessageBus, Test);
var context = new TestContext(TestClass, ConstructorArguments, TestMethod, TestMethodArguments, output);
var lifecycleHooks = GetLifecycleHooks(testClassInstance, TestClass, TestMethod);
await Aggregator.RunAsync(async() =>
{
foreach (var lifecycleHook in lifecycleHooks)
{
await lifecycleHook.OnTestStartAsync(context, CancellationTokenSource.Token);
}
});
var time = await base.InvokeTestMethodAsync(testClassInstance);
await Aggregator.RunAsync(async() =>
{
var exception = Aggregator.HasExceptions ? Aggregator.ToException() : null;
foreach (var lifecycleHook in lifecycleHooks)
{
await lifecycleHook.OnTestEndAsync(context, exception, CancellationTokenSource.Token);
}
});
return(time);
}
protected async override Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator)
{
var output = string.Empty;
TestOutputHelper testOutputHelper = null;
foreach (object obj in this.ConstructorArguments)
{
testOutputHelper = obj as TestOutputHelper;
if (testOutputHelper != null)
{
break;
}
}
if (testOutputHelper != null)
{
testOutputHelper.Initialize(this.MessageBus, this.Test);
}
var tuple = await new StepInvoker(this.step, this.body, aggregator, this.CancellationTokenSource).RunAsync();
this.disposables.AddRange(tuple.Item2);
if (testOutputHelper != null)
{
output = testOutputHelper.Output;
testOutputHelper.Uninitialize();
}
return(Tuple.Create(tuple.Item1, output));
}
/// <inheritdoc/>
protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator)
{
var output = string.Empty;
TestOutputHelper testOutputHelper = null;
foreach (object obj in ConstructorArguments)
{
testOutputHelper = obj as TestOutputHelper;
if (testOutputHelper != null)
{
break;
}
}
if (testOutputHelper != null)
{
testOutputHelper.Initialize(MessageBus, Test);
}
var executionTime = await InvokeTestMethodAsync(aggregator).ConfigureAwait(false);
if (testOutputHelper != null)
{
output = testOutputHelper.Output;
testOutputHelper.Uninitialize();
}
return(Tuple.Create(executionTime, output));
}
protected override async Task <RunSummary> RunTestAsync()
{
var output = new TestOutputHelper();
output.Initialize(this.MessageBus, new XunitTest(this.TestCase, this.DisplayName));
try
{
var exportProvider = await TestUtilities.CreateContainerAsync(this.configuration, output);
var containerWrapper = new TestUtilities.V3ContainerWrapper(exportProvider, this.configuration);
this.TestMethodArguments = new object[] { containerWrapper };
return(await base.RunTestAsync());
}
catch (Exception ex)
{
var t = new XunitTest(this.TestCase, this.DisplayName);
if (!this.MessageBus.QueueMessage(new TestFailed(t, 0, output.Output, ex)))
{
this.CancellationTokenSource.Cancel();
}
return(new RunSummary {
Total = 1, Failed = 1
});
}
}
protected override Task BeforeTestMethodInvokedAsync()
{
if (_output != null)
{
_output.Initialize(MessageBus, Test);
}
return(base.BeforeTestMethodInvokedAsync());
}
protected override async Task <RunSummary> RunTestAsync()
{
var output = new TestOutputHelper();
output.Initialize(this.MessageBus, new XunitTest(this.TestCase, this.DisplayName));
var exportProvider = await TestUtilities.CreateContainerAsync(this.configuration, output);
var containerWrapper = new TestUtilities.V3ContainerWrapper(exportProvider, this.configuration);
this.TestMethodArguments = new object[] { containerWrapper };
return(await base.RunTestAsync());
}
protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator)
{
using (var testLifetimeScope = _testClassLifetimeScope.BeginLifetimeScope(AutofacTestScopes.Test, builder => builder.RegisterModules(TestClass)))
{
TestOutputHelper testOutputHelper = testLifetimeScope.Resolve <TestOutputHelper>();
testOutputHelper.Initialize(MessageBus, Test);
decimal seconds = await new AutofacTestInvoker(testLifetimeScope, Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments,
BeforeAfterAttributes, aggregator, CancellationTokenSource).RunAsync();
string output = testOutputHelper.Output;
testOutputHelper.Uninitialize();
return(Tuple.Create(seconds, output));
}
}
protected override object CreateTestClass()
{
var testClass = base.CreateTestClass();
if (testClass is LoggedTest loggedTestClass)
{
var classType = loggedTestClass.GetType();
var logLevelAttribute = TestMethod.GetCustomAttribute <LogLevelAttribute>() as LogLevelAttribute;
var testName = TestMethodArguments.Aggregate(TestMethod.Name, (a, b) => $"{a}-{(b ?? "null")}");
// Try resolving ITestOutputHelper from constructor arguments
loggedTestClass.TestOutputHelper = ConstructorArguments?.SingleOrDefault(a => typeof(ITestOutputHelper).IsAssignableFrom(a.GetType())) as ITestOutputHelper;
var useShortClassName = TestMethod.DeclaringType.GetCustomAttribute <ShortClassNameAttribute>()
?? TestMethod.DeclaringType.Assembly.GetCustomAttribute <ShortClassNameAttribute>();
var resolvedClassName = useShortClassName == null ? classType.FullName : classType.Name;
// None resolved so create a new one and retain a reference to it for initialization/uninitialization
if (loggedTestClass.TestOutputHelper == null)
{
loggedTestClass.TestOutputHelper = _output = new TestOutputHelper();
_output.Initialize(MessageBus, Test);
}
AssemblyTestLog
.ForAssembly(classType.GetTypeInfo().Assembly)
.StartTestLog(
loggedTestClass.TestOutputHelper,
resolvedClassName,
out var loggerFactory,
logLevelAttribute?.LogLevel ?? LogLevel.Trace,
out var resolvedTestName,
testName);
// internal for testing
loggedTestClass.ResolvedTestMethodName = resolvedTestName;
loggedTestClass.ResolvedTestClassName = resolvedClassName;
loggedTestClass.LoggerFactory = loggerFactory;
loggedTestClass.Logger = loggerFactory.CreateLogger(classType);
loggedTestClass.TestSink = new TestSink();
loggerFactory.AddProvider(new TestLoggerProvider(loggedTestClass.TestSink));
loggedTestClass.AdditionalSetup();
}
return(testClass);
}
public void WriteLine(string outputText, string expected)
{
var output = new TestOutputHelper();
var messageBus = new SpyMessageBus();
var testCase = Mocks.TestCase();
var test = Mocks.Test(testCase, "Test Display Name");
output.Initialize(messageBus, test);
output.WriteLine(outputText);
var message = Assert.Single(messageBus.Messages);
var outputMessage = Assert.IsAssignableFrom <ITestOutput>(message);
Assert.Equal(expected + Environment.NewLine, outputMessage.Output);
Assert.Equal(expected + Environment.NewLine, output.Output);
}
protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator)
{
if (_ownsTestOutputHelper)
{
_testOutputHelper.Initialize(MessageBus, Test);
}
var result = await base.InvokeTestAsync(aggregator);
if (_ownsTestOutputHelper)
{
// Update result with output if we created our own ITestOutputHelper.
// The string returned from this method is what VS displays as the test output.
result = new Tuple <decimal, string>(result.Item1, _testOutputHelper.Output);
_testOutputHelper.Uninitialize();
}
return(result);
}
protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator)
{
_testOutputHelper = FindOutputHelperInArguments(ConstructorArguments);
var shouldOwnTestOutputHelper = _testOutputHelper == null;
if (shouldOwnTestOutputHelper)
{
_testOutputHelper = new TestOutputHelper();
_testOutputHelper.Initialize(MessageBus, Test);
}
var executionResult = await base.InvokeTestAsync(aggregator);
if (shouldOwnTestOutputHelper)
{
executionResult = new Tuple <decimal, string>(executionResult.Item1, _testOutputHelper.Output);
_testOutputHelper.Uninitialize();
}
return(executionResult);
}
protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator)
{
if (_ownsTestOutputHelper)
{
_testOutputHelper.Initialize(MessageBus, Test);
}
var retryAttribute = GetRetryAttribute(TestMethod);
var result = retryAttribute is null
? await base.InvokeTestAsync(aggregator).ConfigureAwait(false)
: await RunTestCaseWithRetryAsync(retryAttribute, aggregator).ConfigureAwait(false);
if (_ownsTestOutputHelper)
{
// Update result with output if we created our own ITestOutputHelper.
// The string returned from this method is what VS displays as the test output.
result = new Tuple <decimal, string>(result.Item1, _testOutputHelper.Output);
_testOutputHelper.Uninitialize();
}
return(result);
}
/// <summary>
/// Invoke the test method.
/// </summary>
/// <param name="aggregator"> The exception aggregator used to run code and collect exceptions. </param>
/// <returns> A tuple that conains the ellapsed time and the test output. </returns>
protected async override Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator)
{
TestContext.EnableOutput = EnableOutput;
if (EnableOutput)
{
TestOutputHelper output = null;
decimal item = default;
string message = default;
try
{
output = new TestOutputHelper();
output.Initialize(base.MessageBus, base.Test);
TestContext.TestOutput = output;
item = await InvokeTestMethodAsync(aggregator);
}
finally
{
if (output != null)
{
message = output.Output;
output.Uninitialize();
TestContext.TestOutput = null;
}
}
return(Tuple.Create(item, message));
}
else
{
decimal item = await InvokeTestMethodAsync(aggregator);
return(Tuple.Create(item, string.Empty));
}
}
protected override async Task <RunSummary> RunTestAsync()
{
var output = new TestOutputHelper();
output.Initialize(this.MessageBus, new XunitTest(this.TestCase, this.DisplayName));
await this.Aggregator.RunAsync(() => this.Timer.AggregateAsync(
async delegate
{
var v3DiscoveryModules = this.GetV3DiscoveryModules();
var resultingCatalogs = new List <ComposableCatalog>(v3DiscoveryModules.Count);
var assemblies = this.assemblyNames.Select(an => Assembly.Load(new AssemblyName(an))).ToList();
foreach (var discoveryModule in v3DiscoveryModules)
{
var partsFromTypes = await discoveryModule.CreatePartsAsync(this.parts);
var partsFromAssemblies = await discoveryModule.CreatePartsAsync(assemblies);
var catalog = TestUtilities.EmptyCatalog
.AddParts(partsFromTypes)
.AddParts(partsFromAssemblies);
resultingCatalogs.Add(catalog);
}
string[] catalogStringRepresentations = resultingCatalogs.Select(catalog =>
{
var writer = new StringWriter();
catalog.ToString(writer);
return(writer.ToString());
}).ToArray();
bool anyStringRepresentationDifferences = false;
for (int i = 1; i < resultingCatalogs.Count; i++)
{
anyStringRepresentationDifferences = PrintDiff(
v3DiscoveryModules[0].GetType().Name,
v3DiscoveryModules[i].GetType().Name,
catalogStringRepresentations[0],
catalogStringRepresentations[i],
output);
}
// Verify that the catalogs are identical.
// The string compare above should have taken care of this (in a more descriptive way),
// but we do this to double-check.
var uniqueCatalogs = resultingCatalogs.Distinct().ToArray();
// Fail the test if ComposableCatalog.Equals returns a different result from string comparison.
Assert.Equal(anyStringRepresentationDifferences, uniqueCatalogs.Length > 1);
if (uniqueCatalogs.Length == 1)
{
////output.WriteLine(catalogStringRepresentations[0]);
}
// For each distinct catalog, create one configuration and verify it meets expectations.
var configurations = new List <CompositionConfiguration>(uniqueCatalogs.Length);
foreach (var uniqueCatalog in uniqueCatalogs)
{
var catalogWithSupport = uniqueCatalog
#if DESKTOP
.WithCompositionService()
#endif
;
// Round-trip the catalog through serialization to verify that as well.
await RoundtripCatalogSerializationAsync(catalogWithSupport, output);
var configuration = CompositionConfiguration.Create(catalogWithSupport);
if (!this.compositionVersions.HasFlag(CompositionEngines.V3AllowConfigurationWithErrors))
{
Assert.Equal(this.expectInvalidConfiguration, !configuration.CompositionErrors.IsEmpty || !catalogWithSupport.DiscoveredParts.DiscoveryErrors.IsEmpty);
}
// Save the configuration in a property so that the engine test that follows can reuse the work we've done.
configurations.Add(configuration);
}
this.ResultingConfigurations = configurations;
}));
var test = new XunitTest(this.TestCase, this.DisplayName);
var runSummary = new RunSummary {
Total = 1, Time = this.Timer.Total
};
IMessageSinkMessage testResultMessage;
if (this.Aggregator.HasExceptions)
{
testResultMessage = new TestFailed(test, this.Timer.Total, output.Output, this.Aggregator.ToException());
runSummary.Failed++;
}
else
{
testResultMessage = new TestPassed(test, this.Timer.Total, output.Output);
this.Passed = true;
}
if (!this.MessageBus.QueueMessage(testResultMessage))
{
this.CancellationTokenSource.Cancel();
}
this.Aggregator.Clear();
return(runSummary);
}
protected override async Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, TestCase.DisplayName); //TODO: this is a pickle, we could use the Compiler/Pickle interfaces from the Gherkin parser
var summary = new RunSummary()
{
Total = 1
};
string output = "";
var gherkinDocument = await this.TestCase.FeatureTypeInfo.GetDocumentAsync();
Scenario scenario = null;
if (gherkinDocument.SpecFlowFeature != null)
{
if (TestCase.IsScenarioOutline)
{
var scenarioOutline = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <ScenarioOutline>().FirstOrDefault(s => s.Name == TestCase.Name);
if (scenarioOutline != null && SpecFlowParserHelper.GetExampleRowById(scenarioOutline, TestCase.ExampleId, out var example, out var exampleRow))
{
scenario = SpecFlowParserHelper.CreateScenario(scenarioOutline, example, exampleRow);
}
}
else
{
scenario = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <Scenario>().FirstOrDefault(s => s.Name == TestCase.Name);
}
}
string skipReason = null;
if (scenario == null)
{
skipReason = $"Unable to find Scenario: {TestCase.DisplayName}";
}
else if (gherkinDocument.SpecFlowFeature.Tags.GetTags().Concat(scenario.Tags.GetTags()).Contains("ignore"))
{
skipReason = "Ignored";
}
if (skipReason != null)
{
summary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(test, skipReason)))
{
CancellationTokenSource.Cancel();
}
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
aggregator.Run(() =>
{
var stopwatch = Stopwatch.StartNew();
testOutputHelper.Initialize(MessageBus, test);
try
{
RunScenario(gherkinDocument, scenario);
}
finally
{
stopwatch.Stop();
summary.Time = (decimal)stopwatch.Elapsed.TotalSeconds;
output = testOutputHelper.Output;
testOutputHelper.Uninitialize();
}
}
);
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
{
testResult = new TestPassed(test, summary.Time, output);
}
else
{
testResult = new TestFailed(test, summary.Time, output, exception);
summary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
{
if (!MessageBus.QueueMessage(testResult))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(test, summary.Time, output)))
{
CancellationTokenSource.Cancel();
}
return(summary);
}
