private async Task <RunSummary> RunMultiEngineTestAsync(CompositionEngines attributesVersion, Type[] parts, IReadOnlyList <string> assemblies, Func <IContainer, Task <RunSummary> > test)
{
try
{
parts = parts ?? new Type[0];
var loadedAssemblies = assemblies != null?assemblies.Select(Assembly.Load).ToImmutableList() : ImmutableList <Assembly> .Empty;
if (attributesVersion.HasFlag(CompositionEngines.V1))
{
return(await test(TestUtilities.CreateContainerV1(loadedAssemblies, parts)));
}
if (attributesVersion.HasFlag(CompositionEngines.V2))
{
return(await test(TestUtilities.CreateContainerV2(loadedAssemblies, parts)));
}
throw new InvalidOperationException();
}
catch (Exception ex)
{
var t = new XunitTest(this.TestCase, this.DisplayName);
if (!this.MessageBus.QueueMessage(new TestFailed(t, 0, null, ex)))
{
this.CancellationTokenSource.Cancel();
}
return(new RunSummary {
Total = 1, Failed = 1
});
}
}
protected override async Task <RunSummary> RunTestAsync()
{
var output = new TestOutputHelper();
output.Initialize(this.MessageBus, new XunitTest(this.TestCase, this.DisplayName));
try
{
var exportProvider = await TestUtilities.CreateContainerAsync(this.configuration, output);
var containerWrapper = new TestUtilities.V3ContainerWrapper(exportProvider, this.configuration);
this.TestMethodArguments = new object[] { containerWrapper };
return(await base.RunTestAsync());
}
catch (Exception ex)
{
var t = new XunitTest(this.TestCase, this.DisplayName);
if (!this.MessageBus.QueueMessage(new TestFailed(t, 0, output.Output, ex)))
{
this.CancellationTokenSource.Cancel();
}
return(new RunSummary {
Total = 1, Failed = 1
});
}
}
private async Task <RunSummary> RunTest()
{
var runSummary = new RunSummary();
Action <ITestResult> onTestFinished = result =>
{
var test = new XunitTest(TestCase, result.DisplayName);
MessageBus.QueueMessage(new TestStarting(test));
var message = GetMessageFromTestResult(test, result);
MessageBus.QueueMessage(message);
MessageBus.QueueMessage(new TestFinished(test, result.ExecutionTime, result.Output));
runSummary.Aggregate(GetRunSummary(result));
};
using (var channelFactory = CreateTestServiceChannelFactory(_Id, onTestFinished))
{
Func <ITestService, Task> action = async service =>
{
try
{
await service.RunTest(
TestCase.Method.Type.Assembly.AssemblyPath,
TestCase.Method.Type.Name,
TestCase.Method.Name);
}
catch (FaultException <TestExecutionFault> fault)
{
throw new TestExecutionException(fault.Detail.Message, fault);
}
};
await Common.ExecuteWithChannel(channelFactory, action);
return(runSummary);
}
}
protected override Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, TestCase.DisplayName);
var summary = new RunSummary {
Total = 1
};
if (!MessageBus.QueueMessage(new TestStarting(test)))
{
CancellationTokenSource.Cancel();
}
else
{
summary.Failed = 1;
var testFailed = exception != null
? new TestFailed(test, 0, output, exception)
: new TestFailed(test, 0, output, exceptionTypes, exceptionMessages, exceptionStackTraces, exceptionParenIndices);
if (!MessageBus.QueueMessage(testFailed))
{
CancellationTokenSource.Cancel();
}
if (!MessageBus.QueueMessage(new TestFinished(test, 0, output)))
{
CancellationTokenSource.Cancel();
}
}
return(Task.FromResult(summary));
}
protected override async Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, DisplayName);
var aggregator = new ExceptionAggregator(Aggregator);
var runner = new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, aggregator, CancellationTokenSource);
return(await KuduXunitTestRunnerUtils.RunTestAsync(runner, MessageBus, aggregator));
}
public void Run(Type type, XunitTest xunitTest)
{
foreach (var prop in type.GetProperties().ToArray())
{
Assert.Null(prop.GetValue(xunitTest));
prop.SetValue(xunitTest, new object());
}
}
private XunitTestRunner CreateTestRunner(XunitTest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo methodToRun, object[] convertedDataRow, string skipReason, IReadOnlyList <BeforeAfterTestAttribute> beforeAfterAttributes, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
{
#if DNX
var constructor = typeof(XunitTestRunner).GetConstructors().First(c => c.GetParameters().Count() == 10);
return((XunitTestRunner)constructor.Invoke(new object[] { test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource }));
#else
return(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
#endif
}
protected override async Task AfterTestCaseStartingAsync()
{
await base.AfterTestCaseStartingAsync();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = SerializationHelper.GetType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(_diagnosticMessageSink, discovererType);
foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method))
{
_toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes);
string theoryDisplayName;
if (convertedDataRow.Length == 1 && convertedDataRow[0] is Test.IReadableTestCase)
{
theoryDisplayName = ((Test.IReadableTestCase)convertedDataRow[0]).TestCase();
}
else
{
theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName, convertedDataRow, resolvedTypes);
}
var test = new XunitTest(TestCase, theoryDisplayName);
var skipReason = SkipReason ?? dataAttribute.GetNamedArgument <string>("Skip");
_testRunners.Add(new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
// Stash the exception so we can surface it during RunTestAsync
_dataDiscoveryException = ex;
}
}
protected override async Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, DisplayName);
var aggregator = new ExceptionAggregator(Aggregator);
var disableRetry = ((KuduXunitTestCase)TestCase).DisableRetry;
if (!disableRetry)
{
var value = ConfigurationManager.AppSettings["DisableRetry"];
disableRetry = string.IsNullOrEmpty(value) || bool.Parse(value);
}
var runner = new XunitTestRunner(test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, aggregator, CancellationTokenSource);
return(await KuduXunitTestRunnerUtils.RunTestAsync(runner, MessageBus, aggregator, disableRetry));
}
private void CreateTheorySpecificTestRunners()
{
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = LoadType(args[1], args[0]);
var discoverer = ExtensibilityPointFactory.GetDataDiscoverer(_diagnosticMessageSink, discovererType);
foreach (var dataRow in discoverer.GetData(dataAttribute, TestCase.TestMethod.Method))
{
_toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
var convertedDataRow = Reflector.ConvertArguments(dataRow, parameterTypes);
var theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName, convertedDataRow, resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
_testRunners.Add(new ScenarioTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun,
convertedDataRow, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource));
}
}
}
catch (Exception ex)
{
// Stash the exception so we can surface it during RunTestAsync
_dataDiscoveryException = ex;
}
}
private RunSummary RunTest_DataDiscoveryException()
{
var test = new XunitTest(TestCase, DisplayName);
if (!MessageBus.QueueMessage(new TestStarting(test)))
{
CancellationTokenSource.Cancel();
}
else if (!MessageBus.QueueMessage(new TestFailed(test, 0, null, Unwrap(_dataDiscoveryException))))
{
CancellationTokenSource.Cancel();
}
if (!MessageBus.QueueMessage(new TestFinished(test, 0, null)))
{
CancellationTokenSource.Cancel();
}
return(new RunSummary {
Total = 1, Failed = 1
});
}
protected override Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, DisplayName);
return(new PartialTrustTestRunner(test, MessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, new ExceptionAggregator(Aggregator), CancellationTokenSource).RunAsync());
}
public MigrationsTestRunner(XunitTest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList <BeforeAfterTestAttribute> beforeAfterAttributes, ExceptionAggregator exceptionAggregator, CancellationTokenSource cancellationTokenSource, DatabaseProvider provider, ProgrammingLanguage language)
: base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, exceptionAggregator, cancellationTokenSource)
{
_provider = provider;
_language = language;
}
protected override async Task <RunSummary> RunTestAsync()
{
var output = new TestOutputHelper();
output.Initialize(this.MessageBus, new XunitTest(this.TestCase, this.DisplayName));
await this.Aggregator.RunAsync(() => this.Timer.AggregateAsync(
async delegate
{
var v3DiscoveryModules = this.GetV3DiscoveryModules();
var resultingCatalogs = new List <ComposableCatalog>(v3DiscoveryModules.Count);
var assemblies = this.assemblyNames.Select(an => Assembly.Load(new AssemblyName(an))).ToList();
foreach (var discoveryModule in v3DiscoveryModules)
{
var partsFromTypes = await discoveryModule.CreatePartsAsync(this.parts);
var partsFromAssemblies = await discoveryModule.CreatePartsAsync(assemblies);
var catalog = TestUtilities.EmptyCatalog
.AddParts(partsFromTypes)
.AddParts(partsFromAssemblies);
resultingCatalogs.Add(catalog);
}
string[] catalogStringRepresentations = resultingCatalogs.Select(catalog =>
{
var writer = new StringWriter();
catalog.ToString(writer);
return(writer.ToString());
}).ToArray();
bool anyStringRepresentationDifferences = false;
for (int i = 1; i < resultingCatalogs.Count; i++)
{
anyStringRepresentationDifferences = PrintDiff(
v3DiscoveryModules[0].GetType().Name,
v3DiscoveryModules[i].GetType().Name,
catalogStringRepresentations[0],
catalogStringRepresentations[i],
output);
}
// Verify that the catalogs are identical.
// The string compare above should have taken care of this (in a more descriptive way),
// but we do this to double-check.
var uniqueCatalogs = resultingCatalogs.Distinct().ToArray();
// Fail the test if ComposableCatalog.Equals returns a different result from string comparison.
Assert.Equal(anyStringRepresentationDifferences, uniqueCatalogs.Length > 1);
if (uniqueCatalogs.Length == 1)
{
////output.WriteLine(catalogStringRepresentations[0]);
}
// For each distinct catalog, create one configuration and verify it meets expectations.
var configurations = new List <CompositionConfiguration>(uniqueCatalogs.Length);
foreach (var uniqueCatalog in uniqueCatalogs)
{
var catalogWithSupport = uniqueCatalog
#if DESKTOP
.WithCompositionService()
#endif
;
// Round-trip the catalog through serialization to verify that as well.
await RoundtripCatalogSerializationAsync(catalogWithSupport, output);
var configuration = CompositionConfiguration.Create(catalogWithSupport);
if (!this.compositionVersions.HasFlag(CompositionEngines.V3AllowConfigurationWithErrors))
{
Assert.Equal(this.expectInvalidConfiguration, !configuration.CompositionErrors.IsEmpty || !catalogWithSupport.DiscoveredParts.DiscoveryErrors.IsEmpty);
}
// Save the configuration in a property so that the engine test that follows can reuse the work we've done.
configurations.Add(configuration);
}
this.ResultingConfigurations = configurations;
}));
var test = new XunitTest(this.TestCase, this.DisplayName);
var runSummary = new RunSummary {
Total = 1, Time = this.Timer.Total
};
IMessageSinkMessage testResultMessage;
if (this.Aggregator.HasExceptions)
{
testResultMessage = new TestFailed(test, this.Timer.Total, output.Output, this.Aggregator.ToException());
runSummary.Failed++;
}
else
{
testResultMessage = new TestPassed(test, this.Timer.Total, output.Output);
this.Passed = true;
}
if (!this.MessageBus.QueueMessage(testResultMessage))
{
this.CancellationTokenSource.Cancel();
}
this.Aggregator.Clear();
return(runSummary);
}
protected override async Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, TestCase.DisplayName); //TODO: this is a pickle, we could use the Compiler/Pickle interfaces from the Gherkin parser
var summary = new RunSummary()
{
Total = 1
};
string output = "";
var gherkinDocument = await this.TestCase.FeatureTypeInfo.GetDocumentAsync();
Scenario scenario = null;
if (gherkinDocument.SpecFlowFeature != null)
{
if (TestCase.IsScenarioOutline)
{
var scenarioOutline = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <ScenarioOutline>().FirstOrDefault(s => s.Name == TestCase.Name);
if (scenarioOutline != null && SpecFlowParserHelper.GetExampleRowById(scenarioOutline, TestCase.ExampleId, out var example, out var exampleRow))
{
scenario = SpecFlowParserHelper.CreateScenario(scenarioOutline, example, exampleRow);
}
}
else
{
scenario = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <Scenario>().FirstOrDefault(s => s.Name == TestCase.Name);
}
}
string skipReason = null;
if (scenario == null)
{
skipReason = $"Unable to find Scenario: {TestCase.DisplayName}";
}
else if (gherkinDocument.SpecFlowFeature.Tags.GetTags().Concat(scenario.Tags.GetTags()).Contains("ignore"))
{
skipReason = "Ignored";
}
if (skipReason != null)
{
summary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(test, skipReason)))
{
CancellationTokenSource.Cancel();
}
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
aggregator.Run(() =>
{
var stopwatch = Stopwatch.StartNew();
testOutputHelper.Initialize(MessageBus, test);
try
{
RunScenario(gherkinDocument, scenario);
}
finally
{
stopwatch.Stop();
summary.Time = (decimal)stopwatch.Elapsed.TotalSeconds;
output = testOutputHelper.Output;
testOutputHelper.Uninitialize();
}
}
);
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
{
testResult = new TestPassed(test, summary.Time, output);
}
else
{
testResult = new TestFailed(test, summary.Time, output, exception);
summary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
{
if (!MessageBus.QueueMessage(testResult))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(test, summary.Time, output)))
{
CancellationTokenSource.Cancel();
}
return(summary);
}
private static XunitTest CreateTypeTest(XunitTypeInfoAdapter typeInfo, ITestClassCommand testClassCommand)
{
XunitTest typeTest = new XunitTest(typeInfo.Target.Name, typeInfo.Target, typeInfo, null);
typeTest.Kind = TestKinds.Fixture;
foreach (XunitMethodInfoAdapter methodInfo in testClassCommand.EnumerateTestMethods())
typeTest.AddChild(CreateMethodTest(typeInfo, methodInfo));
// Add XML documentation.
string xmlDocumentation = typeInfo.Target.GetXmlDocumentation();
if (xmlDocumentation != null)
typeTest.Metadata.SetValue(MetadataKeys.XmlDocumentation, xmlDocumentation);
return typeTest;
}
public override Task <RunSummary> RunAsync(
IMessageSink diagnosticMessageSink,
IMessageBus messageBus,
object[] constructorArguments,
ExceptionAggregator aggregator,
CancellationTokenSource cancellationTokenSource)
{
messageBus.QueueMessage(new TestCaseStarting(this));
var startTime = DateTime.UtcNow;
var test = new XunitTest(this, DisplayName);
var testOutputHelper = constructorArguments
.OfType <TestOutputHelper>()
.FirstOrDefault() ?? new TestOutputHelper();
testOutputHelper.Initialize(messageBus, test);
Task <RunSummary> Fail(Exception ex)
{
var executionTime = (decimal)((DateTime.UtcNow - startTime).TotalSeconds);
messageBus.QueueMessage(new TestFailed(test, executionTime, testOutputHelper !.Output, ex));
return(Task.FromResult(new RunSummary()
{
Failed = 1
}));
}
Task <RunSummary> Pass()
{
var executionTime = (decimal)((DateTime.UtcNow - startTime).TotalSeconds);
messageBus.QueueMessage(new TestPassed(test, executionTime, testOutputHelper !.Output));
return(Task.FromResult(new RunSummary()
{
Total = 1
}));
}
Task <RunSummary> Skip(string reason)
{
messageBus.QueueMessage(new TestSkipped(test, reason));
return(Task.FromResult(new RunSummary()
{
Skipped = 1
}));
}
PropertyInitializationResult?propertyInitResult;
try
{
propertyInitResult = PropertyInitializer.Initialize(
TestMethod.TestClass.Class.ToRuntimeType(),
TestMethod.Method.ToRuntimeMethod(),
constructorArguments,
new DefaultPropertyFactory());
}
catch (Exception exception)
{
return(Fail(exception));
}
if (propertyInitResult.ShouldSkip)
{
return(Skip(propertyInitResult.SkipReason !));
}
try
{
propertyInitResult.Runner.Run(propertyInitResult.Parameters, testOutputHelper);
return(Pass());
}
catch (Exception propertyFailedException)
{
return(Fail(propertyFailedException));
}
}
private async Task ExecuteTestMethod(MethodInfo runtimeMethod, RunSummary runSummary, IEnumerable <IApplicationAttribute> applicationAttributes, object[] dataRow)
{
foreach (var applicationAttribute in applicationAttributes)
{
Application application = null;
Window mainWindow = null;
WindowFixture fixture = null;
try
{
application = applicationAttribute.ProvideApplication(runtimeMethod);
application.WaitWhileBusy();
mainWindow = null;
if (!string.IsNullOrEmpty(applicationAttribute.Window))
{
mainWindow = application.GetWindow(applicationAttribute.Window);
}
else
{
mainWindow = application.GetWindows().FirstOrDefault();
}
if (mainWindow == null)
{
throw new Exception("Could not locate main window " + applicationAttribute.Window);
}
fixture = new WindowFixture(application, mainWindow);
ITypeInfo[] resolvedTypes = null;
var methodToRun = runtimeMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
List <object> parameterList = new List <object>();
var parameters = methodToRun.GetParameters().ToArray();
object initializerReturn = null;
int dataRowIndex = 0;
for (int i = 0; i < parameters.Length; i++)
{
var parameter = parameters[i];
var attributes = parameter.GetCustomAttributes(true);
if (parameter.ParameterType == typeof(Window))
{
parameterList.Add(mainWindow);
}
else if (parameter.ParameterType == typeof(IWindowFixture))
{
parameterList.Add(fixture);
}
else if (attributes.Any(a => a is GenerateAttribute))
{
var generateAttribute = (GenerateAttribute)attributes.First(a => a is GenerateAttribute);
InitializeCustomAttribute(generateAttribute, runtimeMethod, parameter);
var constraintName = generateAttribute.ConstraintName ?? parameter.Name;
var min = generateAttribute.Min;
var max = generateAttribute.Max;
var value = fixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max });
parameterList.Add(value);
}
else if (attributes.Any(a => a is LocateAttribute))
{
var locateAttribute = (LocateAttribute)attributes.First(a => a is LocateAttribute);
InitializeCustomAttribute(locateAttribute, runtimeMethod, parameter);
var value = locateAttribute.Value;
if (value == null)
{
value = fixture.Data.Generate(new SimpleFixture.DataRequest(null,
fixture.Data,
parameter.ParameterType,
parameter.Name,
false,
null,
null));
}
parameterList.Add(value);
}
else if (attributes.Any(a => a is FreezeAttribute))
{
var freeze = (FreezeAttribute)attributes.FirstOrDefault(a => a is FreezeAttribute);
InitializeCustomAttribute(freeze, runtimeMethod, parameter);
var value = freeze.Value;
if (value == null)
{
var constraintName = freeze.ConstraintName ?? parameter.Name;
var min = freeze.Min;
var max = freeze.Max;
value = fixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max });
}
parameterList.Add(value);
object lastObject = parameterList.Last();
var closedFreezeMethod =
FreezeMethod.MakeGenericMethod(lastObject.GetType());
closedFreezeMethod.Invoke(null, new object[] { fixture.Data, value, freeze.For });
}
else if (initializerReturn != null && parameter.ParameterType == initializerReturn.GetType())
{
parameterList.Add(initializerReturn);
initializerReturn = null;
}
else if (dataRowIndex < dataRow.Length)
{
var dataValue = dataRow[dataRowIndex];
dataRowIndex++;
parameterList.Add(dataValue);
}
else
{
var value = fixture.Data.Generate(parameter.ParameterType, parameter.Name);
parameterList.Add(value);
}
}
var convertedDataRow = Reflector.ConvertArguments(parameterList.ToArray(), parameters.Select(p => p.ParameterType).ToArray());
var theoryDisplayName =
TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName + " " + applicationAttribute.Application,
dataRow,
resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
var skipReason = SkipReason;
XunitTestRunner testRunner = CreateTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource);
runSummary.Aggregate(await testRunner.RunAsync());
}
catch (Exception exp)
{
Aggregator.Add(exp);
}
finally
{
var timer = new ExecutionTimer();
timer.Aggregate(() => DisposeOfData(application, mainWindow, fixture, dataRow));
runSummary.Time += timer.Total;
}
}
}
public PartialTrustTestRunner(XunitTest test, IMessageBus messageBus, Type testClass, object[] constructorArguments, MethodInfo testMethod, object[] testMethodArguments, string skipReason, IReadOnlyList <BeforeAfterTestAttribute> beforeAfterAttributes, ExceptionAggregator exceptionAggregator, CancellationTokenSource cancellationTokenSource)
: base(test, messageBus, testClass, constructorArguments, testMethod, testMethodArguments, skipReason, beforeAfterAttributes, exceptionAggregator, cancellationTokenSource)
{
}
private async Task ExecuteTestMethod(MethodInfo runtimeMethod, RunSummary runSummary, IEnumerable <WebDriverAttribute> driverAttributes, object[] dataRow)
{
foreach (var driverAttribute in driverAttributes)
{
foreach (var driver in driverAttribute.GetDrivers(runtimeMethod))
{
Fixture newFixture = null;
object initializerReturn = null;
ITypeInfo[] resolvedTypes = null;
var methodToRun = runtimeMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
List <object> parameterList = new List <object>();
var parameters = methodToRun.GetParameters().ToArray();
try
{
newFixture = FixtureCreationAttribute.GetNewFixture(driver, runtimeMethod);
var initializeDataAttributes = ReflectionHelper.GetAttributes <FixtureInitializationAttribute>(runtimeMethod);
foreach (var initializeDataAttribute in initializeDataAttributes)
{
if (initializeDataAttribute is IMethodInfoAware)
{
#if DNX
var property = initializeDataAttribute.GetType().GetRuntimeProperty("Method");
property.SetValue(initializeDataAttribute, runtimeMethod);
#else
((IMethodInfoAware)initializeDataAttribute).Method = runtimeMethod;
#endif
}
initializeDataAttribute.Initialize(newFixture.Data);
}
var initializeAttribute = ReflectionHelper.GetAttribute <IFixtureInitializationAttribute>(runtimeMethod);
if (initializeAttribute != null)
{
initializerReturn = initializeAttribute.Initialize(runtimeMethod, newFixture);
}
int dataRowIndex = 0;
for (int i = 0; i < parameters.Length; i++)
{
var parameter = parameters[i];
var attributes = parameter.GetCustomAttributes(true);
if (parameter.ParameterType == typeof(IWebDriver))
{
parameterList.Add(driver);
}
else if (parameter.ParameterType == typeof(Fixture))
{
parameterList.Add(newFixture);
}
else if (attributes.Any(a => a is GenerateAttribute))
{
var generateAttribute = (GenerateAttribute)attributes.First(a => a is GenerateAttribute);
InitializeCustomAttribute(generateAttribute, runtimeMethod, parameter);
var constraintName = generateAttribute.ConstraintName ?? parameter.Name;
var min = generateAttribute.Min;
var max = generateAttribute.Max;
var value = newFixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max });
parameterList.Add(value);
}
else if (attributes.Any(a => a is LocateAttribute))
{
var locateAttribute = (LocateAttribute)attributes.First(a => a is LocateAttribute);
InitializeCustomAttribute(locateAttribute, runtimeMethod, parameter);
var value = locateAttribute.Value;
if (value == null)
{
value = newFixture.Data.Generate(new SimpleFixture.DataRequest(null,
newFixture.Data,
parameter.ParameterType,
parameter.Name,
false,
null,
null));
}
parameterList.Add(value);
}
else if (attributes.Any(a => a is FreezeAttribute))
{
var freeze = (FreezeAttribute)attributes.FirstOrDefault(a => a is FreezeAttribute);
InitializeCustomAttribute(freeze, runtimeMethod, parameter);
var value = freeze.Value;
if (value == null)
{
var constraintName = freeze.ConstraintName ?? parameter.Name;
var min = freeze.Min;
var max = freeze.Max;
value = newFixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max });
}
parameterList.Add(value);
object lastObject = parameterList.Last();
var closedFreezeMethod =
FreezeMethod.MakeGenericMethod(lastObject.GetType());
closedFreezeMethod.Invoke(null, new object[] { newFixture.Data, value, freeze.For });
}
else if (initializerReturn != null && parameter.ParameterType == initializerReturn.GetType())
{
parameterList.Add(initializerReturn);
initializerReturn = null;
}
else if (dataRowIndex < dataRow.Length)
{
var dataValue = dataRow[dataRowIndex];
dataRowIndex++;
parameterList.Add(dataValue);
}
else
{
var value = newFixture.Data.Generate(parameter.ParameterType, parameter.Name);
parameterList.Add(value);
}
}
}
catch (Exception exp)
{
Aggregator.Add(exp);
}
var convertedDataRow = Reflector.ConvertArguments(parameterList.ToArray(), parameters.Select(p => p.ParameterType).ToArray());
var theoryDisplayName =
TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName + " " + GetDriverName(driver),
dataRow,
resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
var skipReason = SkipReason;
XunitTestRunner testRunner = CreateTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource);
runSummary.Aggregate(await testRunner.RunAsync());
var timer = new ExecutionTimer();
timer.Aggregate(() => DisposeOfData(driverAttribute, driver, newFixture, dataRow));
runSummary.Time += timer.Total;
}
}
}
private RunSummary FailBecauseOfException(Exception exception)
{
var test = new XunitTest(new TestCaseWithoutTraits(TestCase), DisplayName);
return(FailTestBecauseOfException(test, exception));
}
protected override async Task <RunSummary> RunTestAsync()
{
var test = new XunitTest(TestCase, TestCase.DisplayName); //TODO: this is a pickle, we could use the Compiler/Pickle interfaces from the Gherkin parser
var summary = new RunSummary()
{
Total = 1
};
var output = new StringBuilder();
var gherkinDocument = await SpecFlowParserHelper.ParseSpecFlowDocumentAsync(TestCase.FeatureFile.FeatureFilePath);
Scenario scenario = null;
if (gherkinDocument.SpecFlowFeature != null)
{
if (TestCase.IsScenarioOutline)
{
var scenarioOutline = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <ScenarioOutline>().FirstOrDefault(s => s.Name == TestCase.Name);
Examples example = null;
Gherkin.Ast.TableRow exampleRow = null;
if (scenarioOutline != null && SpecFlowParserHelper.GetExampleRowById(scenarioOutline, TestCase.ExampleId, out example, out exampleRow))
{
scenario = SpecFlowParserHelper.CreateScenario(scenarioOutline, example, exampleRow);
}
}
else
{
scenario = gherkinDocument.SpecFlowFeature.ScenarioDefinitions.OfType <Scenario>().FirstOrDefault(s => s.Name == TestCase.Name);
}
}
string skipReason = null;
if (scenario == null)
{
skipReason = $"Unable to find Scenario: {TestCase.DisplayName}";
}
else if (gherkinDocument.SpecFlowFeature.Tags.GetTags().Concat(scenario.Tags.GetTags()).Contains("ignore"))
{
skipReason = "Ignored";
}
if (skipReason != null)
{
summary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(test, skipReason)))
{
CancellationTokenSource.Cancel();
}
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
aggregator.Run(() => RunScenario(gherkinDocument, scenario, output));
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
{
testResult = new TestPassed(test, summary.Time, output.ToString());
}
else
{
testResult = new TestFailed(test, summary.Time, output.ToString(), exception);
summary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
{
if (!MessageBus.QueueMessage(testResult))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(test, summary.Time, output.ToString())))
{
CancellationTokenSource.Cancel();
}
return(summary);
}
private static XunitTest CreateMethodTest(XunitTypeInfoAdapter typeInfo, XunitMethodInfoAdapter methodInfo)
{
XunitTest methodTest = new XunitTest(methodInfo.Name, methodInfo.Target, typeInfo, methodInfo);
methodTest.Kind = TestKinds.Test;
methodTest.IsTestCase = true;
// Add skip reason.
if (XunitMethodUtility.IsSkip(methodInfo))
{
string skipReason = XunitMethodUtility.GetSkipReason(methodInfo);
if (skipReason != null)
methodTest.Metadata.SetValue(MetadataKeys.IgnoreReason, skipReason);
}
// Add traits.
if (XunitMethodUtility.HasTraits(methodInfo))
{
XunitMethodUtility.GetTraits(methodInfo).ForEach((key, value) =>
methodTest.Metadata.Add(key ?? @"", value ?? @""));
}
// Add XML documentation.
string xmlDocumentation = methodInfo.Target.GetXmlDocumentation();
if (xmlDocumentation != null)
methodTest.Metadata.SetValue(MetadataKeys.XmlDocumentation, xmlDocumentation);
return methodTest;
}
protected override async Task AfterTestCaseStartingAsync()
{
await base.AfterTestCaseStartingAsync();
try
{
var dataAttributes = TestCase.TestMethod.Method.GetCustomAttributes(typeof(DataAttribute));
foreach (var dataAttribute in dataAttributes)
{
var discovererAttribute = dataAttribute.GetCustomAttributes(typeof(DataDiscovererAttribute)).First();
var args = discovererAttribute.GetConstructorArguments().Cast <string>().ToList();
var discovererType = Type.GetType($"{args[0]}, {args[1]}");
if (discovererType == null)
{
if (dataAttribute is IReflectionAttributeInfo reflectionAttribute)
{
Aggregator.Add(new InvalidOperationException($"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name} does not exist."));
}
else
{
Aggregator.Add(new InvalidOperationException($"A data discoverer specified on {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name} does not exist."));
}
continue;
}
IDataDiscoverer discoverer;
try
{
discoverer = ExtensibilityPointFactory.GetDataDiscoverer(_diagnosticMessageSink, discovererType);
}
catch (InvalidCastException)
{
Aggregator.Add(dataAttribute is IReflectionAttributeInfo reflectionAttribute
? new InvalidOperationException(
$"Data discoverer specified for {reflectionAttribute.Attribute.GetType()} on {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name} does not implement IDataDiscoverer.")
: new InvalidOperationException(
$"A data discoverer specified on {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name} does not implement IDataDiscoverer."));
continue;
}
var data = discoverer.GetData(dataAttribute, TestCase.TestMethod.Method);
if (data == null)
{
Aggregator.Add(new InvalidOperationException($"Test data returned null for {TestCase.TestMethod.TestClass.Class.Name}.{TestCase.TestMethod.Method.Name}. Make sure it is statically initialized before this test method is called."));
continue;
}
foreach (var dataRow in data)
{
_toDispose.AddRange(dataRow.OfType <IDisposable>());
ITypeInfo[] resolvedTypes = null;
var methodToRun = TestMethod;
var convertedDataRow = methodToRun.ResolveMethodArguments(dataRow);
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(convertedDataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
var parameterTypes = methodToRun.GetParameters().Select(p => p.ParameterType).ToArray();
convertedDataRow = Reflector.ConvertArguments(convertedDataRow, parameterTypes);
object compatibilityLevel;
if (TestMethodArguments[0] is CompatibilityLevel level)
{
compatibilityLevel = level;
}
else
{
compatibilityLevel = Enum.Parse(typeof(CompatibilityLevel), (string)TestMethodArguments[0]);
}
var finalDataRow = convertedDataRow.Concat(new [] { compatibilityLevel }).ToArray();
var theoryDisplayName = TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName, finalDataRow, resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
var skipReason = SkipReason ?? dataAttribute.GetNamedArgument <string>("Skip");
var testRunner = new DataCompatibilityRangeTestRunner(test, MessageBus, TestClass, ConstructorArguments,
methodToRun, finalDataRow, skipReason, BeforeAfterAttributes, Aggregator,
CancellationTokenSource);
_testRunners.Add(testRunner);
}
}
}
catch (Exception ex)
{
_dataDiscoveryException = ex;
}
}
