protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator) { // copy from XunitTestRunner var output = string.Empty; TestOutputHelper testOutputHelper = null; foreach (object obj in ConstructorArguments) { testOutputHelper = obj as TestOutputHelper; if (testOutputHelper != null) { break; } } if (testOutputHelper != null) { testOutputHelper.Initialize(MessageBus, Test); } var executionTime = await InvokeTestMethodAsync(aggregator); if (testOutputHelper != null) { output = testOutputHelper.Output; testOutputHelper.Uninitialize(); } // save the exceptions that were thrown during the test, in case this is needed ExceptionAggregator?.Aggregate(aggregator); return(Tuple.Create(executionTime, output)); }
/// <inheritdoc /> protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator) { var output = string.Empty; TestOutputHelper testOutputHelper = null; foreach (var obj in ConstructorArguments) { testOutputHelper = obj as TestOutputHelper; if (testOutputHelper != null) { break; } } testOutputHelper?.Initialize(MessageBus, Test); //TODO: Add execution time, if strictly necessary - the NBench values are certainly more accurate. var executionTime = await InvokeTestMethodAsync(aggregator); if (testOutputHelper != null) { output = testOutputHelper.Output; testOutputHelper.Uninitialize(); } return(Tuple.Create(executionTime, output)); }
public void WriteLine(string outputText, string expected) { var output = new TestOutputHelper(); var messageBus = new SpyMessageBus(); var test = Substitute.For <_ITest>(); test.UniqueID.Returns("test-id"); test.TestCase.UniqueID.Returns("case-id"); test.TestCase.TestMethod.UniqueID.Returns("method-id"); test.TestCase.TestMethod.TestClass.UniqueID.Returns("class-id"); test.TestCase.TestMethod.TestClass.TestCollection.UniqueID.Returns("coll-id"); test.TestCase.TestMethod.TestClass.TestCollection.TestAssembly.UniqueID.Returns("asm-id"); output.Initialize(messageBus, test); output.WriteLine(outputText); var message = Assert.Single(messageBus.Messages); var outputMessage = Assert.IsType <_TestOutput>(message); Assert.Equal("asm-id", outputMessage.AssemblyUniqueID); Assert.Equal(expected + Environment.NewLine, outputMessage.Output); Assert.Equal("case-id", outputMessage.TestCaseUniqueID); Assert.Equal("class-id", outputMessage.TestClassUniqueID); Assert.Equal("coll-id", outputMessage.TestCollectionUniqueID); Assert.Equal("method-id", outputMessage.TestMethodUniqueID); Assert.Equal("test-id", outputMessage.TestUniqueID); Assert.Equal(expected + Environment.NewLine, output.Output); }
protected override async Task <decimal> InvokeTestMethodAsync(object testClassInstance) { var output = new TestOutputHelper(); output.Initialize(MessageBus, Test); var context = new TestContext(TestClass, ConstructorArguments, TestMethod, TestMethodArguments, output); var lifecycleHooks = GetLifecycleHooks(testClassInstance, TestClass, TestMethod); await Aggregator.RunAsync(async() => { foreach (var lifecycleHook in lifecycleHooks) { await lifecycleHook.OnTestStartAsync(context, CancellationTokenSource.Token); } }); var time = await base.InvokeTestMethodAsync(testClassInstance); await Aggregator.RunAsync(async() => { var exception = Aggregator.HasExceptions ? Aggregator.ToException() : null; foreach (var lifecycleHook in lifecycleHooks) { await lifecycleHook.OnTestEndAsync(context, exception, CancellationTokenSource.Token); } }); return(time); }
protected async override Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator) { var output = string.Empty; TestOutputHelper testOutputHelper = null; foreach (object obj in this.ConstructorArguments) { testOutputHelper = obj as TestOutputHelper; if (testOutputHelper != null) { break; } } if (testOutputHelper != null) { testOutputHelper.Initialize(this.MessageBus, this.Test); } var tuple = await new StepInvoker(this.step, this.body, aggregator, this.CancellationTokenSource).RunAsync(); this.disposables.AddRange(tuple.Item2); if (testOutputHelper != null) { output = testOutputHelper.Output; testOutputHelper.Uninitialize(); } return(Tuple.Create(tuple.Item1, output)); }
/// <inheritdoc/> protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator) { var output = string.Empty; TestOutputHelper testOutputHelper = null; foreach (object obj in ConstructorArguments) { testOutputHelper = obj as TestOutputHelper; if (testOutputHelper != null) { break; } } if (testOutputHelper != null) { testOutputHelper.Initialize(MessageBus, Test); } var executionTime = await InvokeTestMethodAsync(aggregator).ConfigureAwait(false); if (testOutputHelper != null) { output = testOutputHelper.Output; testOutputHelper.Uninitialize(); } return(Tuple.Create(executionTime, output)); }
protected override async Task <RunSummary> RunTestAsync() { var output = new TestOutputHelper(); output.Initialize(this.MessageBus, new XunitTest(this.TestCase, this.DisplayName)); try { var exportProvider = await TestUtilities.CreateContainerAsync(this.configuration, output); var containerWrapper = new TestUtilities.V3ContainerWrapper(exportProvider, this.configuration); this.TestMethodArguments = new object[] { containerWrapper }; return(await base.RunTestAsync()); } catch (Exception ex) { var t = new XunitTest(this.TestCase, this.DisplayName); if (!this.MessageBus.QueueMessage(new TestFailed(t, 0, output.Output, ex))) { this.CancellationTokenSource.Cancel(); } return(new RunSummary { Total = 1, Failed = 1 }); } }
protected override Task BeforeTestMethodInvokedAsync() { if (_output != null) { _output.Initialize(MessageBus, Test); } return(base.BeforeTestMethodInvokedAsync()); }
protected override async Task <RunSummary> RunTestAsync() { var output = new TestOutputHelper(); output.Initialize(this.MessageBus, new XunitTest(this.TestCase, this.DisplayName)); var exportProvider = await TestUtilities.CreateContainerAsync(this.configuration, output); var containerWrapper = new TestUtilities.V3ContainerWrapper(exportProvider, this.configuration); this.TestMethodArguments = new object[] { containerWrapper }; return(await base.RunTestAsync()); }
protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator) { using (var testLifetimeScope = _testClassLifetimeScope.BeginLifetimeScope(AutofacTestScopes.Test, builder => builder.RegisterModules(TestClass))) { TestOutputHelper testOutputHelper = testLifetimeScope.Resolve <TestOutputHelper>(); testOutputHelper.Initialize(MessageBus, Test); decimal seconds = await new AutofacTestInvoker(testLifetimeScope, Test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, BeforeAfterAttributes, aggregator, CancellationTokenSource).RunAsync(); string output = testOutputHelper.Output; testOutputHelper.Uninitialize(); return(Tuple.Create(seconds, output)); } }
protected override object CreateTestClass() { var testClass = base.CreateTestClass(); if (testClass is LoggedTest loggedTestClass) { var classType = loggedTestClass.GetType(); var logLevelAttribute = TestMethod.GetCustomAttribute <LogLevelAttribute>() as LogLevelAttribute; var testName = TestMethodArguments.Aggregate(TestMethod.Name, (a, b) => $"{a}-{(b ?? "null")}"); // Try resolving ITestOutputHelper from constructor arguments loggedTestClass.TestOutputHelper = ConstructorArguments?.SingleOrDefault(a => typeof(ITestOutputHelper).IsAssignableFrom(a.GetType())) as ITestOutputHelper; var useShortClassName = TestMethod.DeclaringType.GetCustomAttribute <ShortClassNameAttribute>() ?? TestMethod.DeclaringType.Assembly.GetCustomAttribute <ShortClassNameAttribute>(); var resolvedClassName = useShortClassName == null ? classType.FullName : classType.Name; // None resolved so create a new one and retain a reference to it for initialization/uninitialization if (loggedTestClass.TestOutputHelper == null) { loggedTestClass.TestOutputHelper = _output = new TestOutputHelper(); _output.Initialize(MessageBus, Test); } AssemblyTestLog .ForAssembly(classType.GetTypeInfo().Assembly) .StartTestLog( loggedTestClass.TestOutputHelper, resolvedClassName, out var loggerFactory, logLevelAttribute?.LogLevel ?? LogLevel.Trace, out var resolvedTestName, testName); // internal for testing loggedTestClass.ResolvedTestMethodName = resolvedTestName; loggedTestClass.ResolvedTestClassName = resolvedClassName; loggedTestClass.LoggerFactory = loggerFactory; loggedTestClass.Logger = loggerFactory.CreateLogger(classType); loggedTestClass.TestSink = new TestSink(); loggerFactory.AddProvider(new TestLoggerProvider(loggedTestClass.TestSink)); loggedTestClass.AdditionalSetup(); } return(testClass); }
public void WriteLine(string outputText, string expected) { var output = new TestOutputHelper(); var messageBus = new SpyMessageBus(); var testCase = Mocks.TestCase(); var test = Mocks.Test(testCase, "Test Display Name"); output.Initialize(messageBus, test); output.WriteLine(outputText); var message = Assert.Single(messageBus.Messages); var outputMessage = Assert.IsAssignableFrom <ITestOutput>(message); Assert.Equal(expected + Environment.NewLine, outputMessage.Output); Assert.Equal(expected + Environment.NewLine, output.Output); }
protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator) { if (_ownsTestOutputHelper) { _testOutputHelper.Initialize(MessageBus, Test); } var result = await base.InvokeTestAsync(aggregator); if (_ownsTestOutputHelper) { // Update result with output if we created our own ITestOutputHelper. // The string returned from this method is what VS displays as the test output. result = new Tuple <decimal, string>(result.Item1, _testOutputHelper.Output); _testOutputHelper.Uninitialize(); } return(result); }
protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator) { _testOutputHelper = FindOutputHelperInArguments(ConstructorArguments); var shouldOwnTestOutputHelper = _testOutputHelper == null; if (shouldOwnTestOutputHelper) { _testOutputHelper = new TestOutputHelper(); _testOutputHelper.Initialize(MessageBus, Test); } var executionResult = await base.InvokeTestAsync(aggregator); if (shouldOwnTestOutputHelper) { executionResult = new Tuple <decimal, string>(executionResult.Item1, _testOutputHelper.Output); _testOutputHelper.Uninitialize(); } return(executionResult); }
protected override async Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator) { if (_ownsTestOutputHelper) { _testOutputHelper.Initialize(MessageBus, Test); } var retryAttribute = GetRetryAttribute(TestMethod); var result = retryAttribute is null ? await base.InvokeTestAsync(aggregator).ConfigureAwait(false) : await RunTestCaseWithRetryAsync(retryAttribute, aggregator).ConfigureAwait(false); if (_ownsTestOutputHelper) { // Update result with output if we created our own ITestOutputHelper. // The string returned from this method is what VS displays as the test output. result = new Tuple <decimal, string>(result.Item1, _testOutputHelper.Output); _testOutputHelper.Uninitialize(); } return(result); }
/// <summary> /// Invoke the test method. /// </summary> /// <param name="aggregator"> The exception aggregator used to run code and collect exceptions. </param> /// <returns> A tuple that conains the ellapsed time and the test output. </returns> protected async override Task <Tuple <decimal, string> > InvokeTestAsync(ExceptionAggregator aggregator) { TestContext.EnableOutput = EnableOutput; if (EnableOutput) { TestOutputHelper output = null; decimal item = default; string message = default; try { output = new TestOutputHelper(); output.Initialize(base.MessageBus, base.Test); TestContext.TestOutput = output; item = await InvokeTestMethodAsync(aggregator); } finally { if (output != null) { message = output.Output; output.Uninitialize(); TestContext.TestOutput = null; } } return(Tuple.Create(item, message)); } else { decimal item = await InvokeTestMethodAsync(aggregator); return(Tuple.Create(item, string.Empty)); } }
protected override async Task <RunSummary> RunTestAsync() { var output = new TestOutputHelper(); output.Initialize(this.MessageBus, new XunitTest(this.TestCase, this.DisplayName)); await this.Aggregator.RunAsync(() => this.Timer.AggregateAsync( async delegate { var v3DiscoveryModules = this.GetV3DiscoveryModules(); var resultingCatalogs = new List <ComposableCatalog>(v3DiscoveryModules.Count); var assemblies = this.assemblyNames.Select(an => Assembly.Load(new AssemblyName(an))).ToList(); foreach (var discoveryModule in v3DiscoveryModules) { var partsFromTypes = await discoveryModule.CreatePartsAsync(this.parts); var partsFromAssemblies = await discoveryModule.CreatePartsAsync(assemblies); var catalog = TestUtilities.EmptyCatalog .AddParts(partsFromTypes) .AddParts(partsFromAssemblies); resultingCatalogs.Add(catalog); } string[] catalogStringRepresentations = resultingCatalogs.Select(catalog => { var writer = new StringWriter(); catalog.ToString(writer); return(writer.ToString()); }).ToArray(); bool anyStringRepresentationDifferences = false; for (int i = 1; i < resultingCatalogs.Count; i++) { anyStringRepresentationDifferences = PrintDiff( v3DiscoveryModules[0].GetType().Name, v3DiscoveryModules[i].GetType().Name, catalogStringRepresentations[0], catalogStringRepresentations[i], output); } // Verify that the catalogs are identical. // The string compare above should have taken care of this (in a more descriptive way), // but we do this to double-check. var uniqueCatalogs = resultingCatalogs.Distinct().ToArray(); // Fail the test if ComposableCatalog.Equals returns a different result from string comparison. Assert.Equal(anyStringRepresentationDifferences, uniqueCatalogs.Length > 1); if (uniqueCatalogs.Length == 1) { ////output.WriteLine(catalogStringRepresentations[0]); } // For each distinct catalog, create one configuration and verify it meets expectations. var configurations = new List <CompositionConfiguration>(uniqueCatalogs.Length); foreach (var uniqueCatalog in uniqueCatalogs) { var catalogWithSupport = uniqueCatalog #if DESKTOP .WithCompositionService() #endif ; // Round-trip the catalog through serialization to verify that as well. await RoundtripCatalogSerializationAsync(catalogWithSupport, output); var configuration = CompositionConfiguration.Create(catalogWithSupport); if (!this.compositionVersions.HasFlag(CompositionEngines.V3AllowConfigurationWithErrors)) { Assert.Equal(this.expectInvalidConfiguration, !configuration.CompositionErrors.IsEmpty || !catalogWithSupport.DiscoveredParts.DiscoveryErrors.IsEmpty); } // Save the configuration in a property so that the engine test that follows can reuse the work we've done. configurations.Add(configuration); } this.ResultingConfigurations = configurations; })); var test = new XunitTest(this.TestCase, this.DisplayName); var runSummary = new RunSummary { Total = 1, Time = this.Timer.Total }; IMessageSinkMessage testResultMessage; if (this.Aggregator.HasExceptions) { testResultMessage = new TestFailed(test, this.Timer.Total, output.Output, this.Aggregator.ToException()); runSummary.Failed++; } else { testResultMessage = new TestPassed(test, this.Timer.Total, output.Output); this.Passed = true; } if (!this.MessageBus.QueueMessage(testResultMessage)) { this.CancellationTokenSource.Cancel(); } this.Aggregator.Clear(); return(runSummary); }
protected override async Task <RunSummary> RunTestAsync() { var test = new XunitTest(TestCase, TestCase.DisplayName); //TODO: this is a pickle, we could use the Compiler/Pickle interfaces from the Gherkin parser var summary = new RunSummary() { Total = 1 }; string output = ""; var gherkinDocument = await this.TestCase.FeatureTypeInfo.GetDocumentAsync(); Scenario scenario = null; if (gherkinDocument.SpecFlowFeature != null) { if (TestCase.IsScenarioOutline) { var scenarioOutline = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <ScenarioOutline>().FirstOrDefault(s => s.Name == TestCase.Name); if (scenarioOutline != null && SpecFlowParserHelper.GetExampleRowById(scenarioOutline, TestCase.ExampleId, out var example, out var exampleRow)) { scenario = SpecFlowParserHelper.CreateScenario(scenarioOutline, example, exampleRow); } } else { scenario = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <Scenario>().FirstOrDefault(s => s.Name == TestCase.Name); } } string skipReason = null; if (scenario == null) { skipReason = $"Unable to find Scenario: {TestCase.DisplayName}"; } else if (gherkinDocument.SpecFlowFeature.Tags.GetTags().Concat(scenario.Tags.GetTags()).Contains("ignore")) { skipReason = "Ignored"; } if (skipReason != null) { summary.Skipped++; if (!MessageBus.QueueMessage(new TestSkipped(test, skipReason))) { CancellationTokenSource.Cancel(); } } else { var aggregator = new ExceptionAggregator(Aggregator); if (!aggregator.HasExceptions) { aggregator.Run(() => { var stopwatch = Stopwatch.StartNew(); testOutputHelper.Initialize(MessageBus, test); try { RunScenario(gherkinDocument, scenario); } finally { stopwatch.Stop(); summary.Time = (decimal)stopwatch.Elapsed.TotalSeconds; output = testOutputHelper.Output; testOutputHelper.Uninitialize(); } } ); } var exception = aggregator.ToException(); TestResultMessage testResult; if (exception == null) { testResult = new TestPassed(test, summary.Time, output); } else { testResult = new TestFailed(test, summary.Time, output, exception); summary.Failed++; } if (!CancellationTokenSource.IsCancellationRequested) { if (!MessageBus.QueueMessage(testResult)) { CancellationTokenSource.Cancel(); } } } if (!MessageBus.QueueMessage(new TestFinished(test, summary.Time, output))) { CancellationTokenSource.Cancel(); } return(summary); }