/// <summary>
/// Adds a run summary's totals into this run summary.
/// </summary>
/// <param name="other">The run summary to be added.</param>
public void Aggregate(RunSummary other)
{
Total += other.Total;
Failed += other.Failed;
Skipped += other.Skipped;
Time += other.Time;
}
protected static void DisplayTestResults(RunSummary summary)
{
var caller =
new StackTrace().GetFrames()
.First(f => typeof(NServiceBusPerformanceTest).IsAssignableFrom(f.GetMethod().DeclaringType.BaseType));
var testCategory =
caller.GetMethod()
.DeclaringType.Namespace.Replace(typeof(NServiceBusPerformanceTest).Namespace + ".", "");
var testCase = caller.GetMethod().Name;
var c = (PerformanceTestContext)summary.RunDescriptor.ScenarioContext;
var messagesPerSecondsProcessed = c.NumberOfTestMessages/
(c.LastMessageProcessedAt - c.FirstMessageProcessedAt).TotalSeconds;
Console.Out.WriteLine("Results: {0} messages, {1} msg/s", c.NumberOfTestMessages,
messagesPerSecondsProcessed);
using (var file = new StreamWriter(".\\PerformanceTestResults.txt", true))
{
file.WriteLine(string.Join(";", summary.RunDescriptor.Key, testCategory, testCase,
c.NumberOfTestMessages, messagesPerSecondsProcessed));
}
Console.Out.WriteLine("##teamcity[buildStatisticValue key='{0}' value='{1:0}']",
summary.RunDescriptor.Key + "." + testCategory + "." + testCase, messagesPerSecondsProcessed);
}
/// <summary>
/// Initializes a new instance of the <see cref="TestClassCallbackHandler" /> class.
/// </summary>
/// <param name="testCases">The test cases that are being run.</param>
/// <param name="messageSink">The message sink to call with the translated results.</param>
public TestClassCallbackHandler(IList<Xunit1TestCase> testCases, IMessageSink messageSink)
: base(lastNodeName: "class")
{
this.handlers = new Dictionary<string, Predicate<XmlNode>> { { "class", OnClass }, { "start", OnStart }, { "test", OnTest } };
this.messageSink = messageSink;
this.testCases = testCases;
this.testCaseResults = new RunSummary();
TestClassResults = new RunSummary();
}
/// <summary>
/// Invokes the Steps given Background, Scenario, and TearDown steps.
/// </summary>
/// <param name="backgroundSteps"></param>
/// <param name="scenarioSteps"></param>
/// <param name="tearDownSteps"></param>
/// <returns></returns>
private async Task <RunSummary> InvokeStepsAsync(ICollection <IStepDefinition> backgroundSteps
, ICollection <IStepDefinition> scenarioSteps, ICollection <IStepDefinition> tearDownSteps)
{
var scenarioTypeInfo = this._scenarioClass.GetTypeInfo();
var filters = scenarioTypeInfo.Assembly.GetCustomAttributes(typeof(Attribute))
.Concat(scenarioTypeInfo.GetCustomAttributes(typeof(Attribute)))
.Concat(this._scenarioMethod.GetCustomAttributes(typeof(Attribute)))
.OfType <IFilter <IStepDefinition> >();
var summary = new RunSummary();
string skipReason = null;
var scenarioRollbacks = new List <(StepContext context, Func <IStepContext, Task> callback)>();
var stepNumber = 0;
foreach (var stepDefinition in filters.Aggregate(backgroundSteps.Concat(scenarioSteps).Concat(tearDownSteps)
, (current, filter) => filter.Filter(current)))
{
stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason;
var stepDisplayName = GetStepDisplayName(this._scenario.DisplayName
, ++stepNumber
, stepDefinition.OnDisplayText?.Invoke(
stepDefinition.Text, stepDefinition.StepDefinitionType)
);
var step = new StepTest(this._scenario, stepDisplayName);
// #1 MWP 2020-07-01 11:56:45 AM: After testing, this should be fine, and better behaved we think.
using (var interceptingBus = new DelegatingMessageBus(
this._messageBus
, message =>
{
if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip)
{
skipReason = $"Failed to execute preceding step: {step.DisplayName}";
}
})
)
{
var stepContext = new StepContext(step);
// TODO: TBD: #1 MWP 2020-07-01 11:57:26 AM: it is an xUnit thing, could possibly be IDisposable itself...
// TODO: TBD: including assumed ownership of the IDisposable messageBus, for instance, TODO: TBD: but that is outside the scope of xWellBehaved...
var stepRunner = new StepTestRunner(
stepContext
, stepDefinition.Body
, step
, interceptingBus
, this._scenarioClass
, this._constructorArguments
, this._scenarioMethod
, this._scenarioMethodArguments
, stepDefinition.SkipReason
, Array.Empty <BeforeAfterTestAttribute>()
, new ExceptionAggregator(this._aggregator)
, this._cancellationTokenSource);
summary.Aggregate(await stepRunner.RunAsync());
// TODO: TBD: could we use disposable?.Dispose() here?
var stepRollbacks = stepContext.Disposables
.Where(disposable => disposable != null)
.Select((Func <IDisposable, Func <IStepContext, Task> >)(disposable => context =>
{
disposable.Dispose();
return(Task.FromResult(0));
}))
.Concat(stepDefinition.Rollbacks)
.Where(onRollback => onRollback != null)
.Select(onRollback => (stepContext, onRollback));
scenarioRollbacks.AddRange(stepRollbacks);
}
}
if (scenarioRollbacks.Any())
{
scenarioRollbacks.Reverse();
var rollbackTimer = new ExecutionTimer();
var rollbackAggregator = new ExceptionAggregator();
// "Teardowns" not to be confused with TearDown versus Background.
foreach (var(context, onRollback) in scenarioRollbacks)
{
await Invoker.Invoke(() => onRollback.Invoke(context), rollbackAggregator, rollbackTimer);
}
summary.Time += rollbackTimer.Total;
if (rollbackAggregator.HasExceptions)
{
summary.Failed++;
summary.Total++;
var stepDisplayName = GetStepDisplayName(this._scenario.DisplayName, ++stepNumber, $"({StepType.Rollback})");
this._messageBus.Queue(new StepTest(this._scenario, stepDisplayName)
, test => new TestFailed(test, rollbackTimer.Total, null, rollbackAggregator.ToException())
, this._cancellationTokenSource);
}
}
return(summary);
}
public void Parse(RunSummary runSummary)
{
_runSummary = runSummary;
}
private RunSummary RunNBenchTest(object testClassInstance)
{
//TODO: It is not strictly required to use a RunSummary at the moment - needs more investigation to see
//if we can provide more useful information via the standard xUnit mechanism. For now, what we have is sufficient.
var summary = new RunSummary();
var discovery = new ReflectionDiscovery(new ActionBenchmarkOutput(report => { }, results =>
{
if (results.Data.Exceptions.Any())
{
throw new AggregateException(results.Data.Exceptions);
}
WriteTestOutput("");
if (results.AssertionResults.Count > 0)
{
//TODO: We should determine the accurate elapsed time at this point, to report it in the xUnit runner.
summary.Time = (decimal)results.Data.StatsByMetric.Values.First().Runs.First().ElapsedSeconds;
foreach (var assertion in results.AssertionResults)
{
//TODO: Maybe it is bubble to bubble this up, and provide the original line number?
Assert.True(assertion.Passed, assertion.Message);
summary.Total++;
if (!assertion.Passed)
{
summary.Failed++;
}
WriteTestOutput(assertion.Message);
WriteTestOutput("");
}
}
else
{
WriteTestOutput("No assertions returned.");
}
WriteTestOutput("");
WriteTestOutput("---------- Measurements ----------");
WriteTestOutput("");
if (results.Data.StatsByMetric.Count > 0)
{
foreach (var measurement in results.Data.StatsByMetric)
{
WriteTestOutput("Metric : " + measurement.Key.ToHumanFriendlyString());
WriteTestOutput("");
WriteTestOutput($"Per Second ( {measurement.Value.Unit} )");
WriteTestOutput($"Average : {measurement.Value.PerSecondStats.Average}");
WriteTestOutput($"Max : {measurement.Value.PerSecondStats.Max}");
WriteTestOutput($"Min : {measurement.Value.PerSecondStats.Min}");
WriteTestOutput($"Std. Deviation : {measurement.Value.PerSecondStats.StandardDeviation}");
WriteTestOutput($"Std. Error : {measurement.Value.PerSecondStats.StandardError}");
WriteTestOutput("");
WriteTestOutput($"Per Test ( {measurement.Value.Unit} )");
WriteTestOutput($"Average : {measurement.Value.Stats.Average}");
WriteTestOutput($"Max : {measurement.Value.Stats.Max}");
WriteTestOutput($"Min : {measurement.Value.Stats.Min}");
WriteTestOutput($"Std. Deviation : {measurement.Value.Stats.StandardDeviation}");
WriteTestOutput($"Std. Error : {measurement.Value.Stats.StandardError}");
WriteTestOutput("");
WriteTestOutput("----------");
WriteTestOutput("");
}
}
else
{
WriteTestOutput("No measurements returned.");
}
}));
var testClassType = TestClass;
//TODO: At the moment this is performing work that is not required, but is pragmatic in that a change is not required to the NBench core.
var benchmarkMetaData = ReflectionDiscovery.CreateBenchmarksForClass(testClassType).First(b => b.Run.InvocationMethod.Name == TestMethod.Name);
try
{
var invoker =
new XUnitReflectionBenchmarkInvoker(benchmarkMetaData, testClassInstance, TestMethodArguments);
var settings = discovery.CreateSettingsForBenchmark(benchmarkMetaData);
var benchmark = new Benchmark(settings, invoker, discovery.Output, discovery.BenchmarkAssertions);
Benchmark.PrepareForRun();
benchmark.Run();
benchmark.Finish();
}
catch (ReflectionTypeLoadException ex)
{
foreach (var e in ex.LoaderExceptions)
{
WriteTestOutput(e.ToString());
}
throw;
}
return(summary);
}
TestableTestCaseRunner(ITestCase testCase, IMessageBus messageBus, ExceptionAggregator aggregator, CancellationTokenSource tokenSource, RunSummary result)
: base(testCase, messageBus, aggregator, tokenSource)
{
this.result = result;
TestCase = testCase;
TokenSource = tokenSource;
}
/// <summary>
/// Gets the hash code
/// </summary>
/// <returns>Hash code</returns>
public override int GetHashCode()
{
unchecked // Overflow is fine, just wrap
{
var hashCode = 41;
// Suitable nullity checks etc, of course :)
if (Class != null)
{
hashCode = hashCode * 59 + Class.GetHashCode();
}
if (Links != null)
{
hashCode = hashCode * 59 + Links.GetHashCode();
}
hashCode = hashCode * 59 + DurationInMillis.GetHashCode();
if (EnQueueTime != null)
{
hashCode = hashCode * 59 + EnQueueTime.GetHashCode();
}
if (EndTime != null)
{
hashCode = hashCode * 59 + EndTime.GetHashCode();
}
hashCode = hashCode * 59 + EstimatedDurationInMillis.GetHashCode();
if (Id != null)
{
hashCode = hashCode * 59 + Id.GetHashCode();
}
if (Organization != null)
{
hashCode = hashCode * 59 + Organization.GetHashCode();
}
if (Pipeline != null)
{
hashCode = hashCode * 59 + Pipeline.GetHashCode();
}
if (Result != null)
{
hashCode = hashCode * 59 + Result.GetHashCode();
}
if (RunSummary != null)
{
hashCode = hashCode * 59 + RunSummary.GetHashCode();
}
if (StartTime != null)
{
hashCode = hashCode * 59 + StartTime.GetHashCode();
}
if (State != null)
{
hashCode = hashCode * 59 + State.GetHashCode();
}
if (Type != null)
{
hashCode = hashCode * 59 + Type.GetHashCode();
}
if (CommitId != null)
{
hashCode = hashCode * 59 + CommitId.GetHashCode();
}
return(hashCode);
}
}
public virtual Task DisposeAsync(RunSummary result, IMessageBus messageBus) => Task.CompletedTask;
private static async Task <RunSummary> RunTestAsyncCore(XunitTestRunner runner,
IMessageBus messageBus,
ExceptionAggregator aggregator)
{
try
{
DelayedMessageBus delayedMessageBus = null;
RunSummary summary = null;
// First run
if (!KuduUtils.DisableRetry)
{
// This is really the only tricky bit: we need to capture and delay messages (since those will
// contain run status) until we know we've decided to accept the final result;
delayedMessageBus = new DelayedMessageBus(messageBus);
runner.SetMessageBus(delayedMessageBus);
summary = await RunTestInternalAsync(runner);
if (delayedMessageBus.TestSkipped)
{
summary.Failed -= 1;
summary.Skipped += 1;
}
// if succeeded
if (summary.Failed == 0 || aggregator.HasExceptions)
{
delayedMessageBus.Flush(false);
return(summary);
}
}
// Final run
var kuduMessageBus = new KuduTraceMessageBus(messageBus);
runner.SetMessageBus(kuduMessageBus);
summary = await RunTestInternalAsync(runner);
if (kuduMessageBus.TestSkipped)
{
summary.Failed -= 1;
summary.Skipped += 1;
}
// flush delay messages
if (delayedMessageBus != null)
{
delayedMessageBus.Flush(summary.Failed == 0 && !aggregator.HasExceptions);
}
return(summary);
}
catch (Exception ex)
{
// this is catastrophic
messageBus.QueueMessage(new TestFailed(runner.GetTest(), 0, null, ex));
return(new RunSummary {
Failed = 1, Total = 1
});
}
finally
{
// set to original
runner.SetMessageBus(messageBus);
}
}
public async Task <RunSummary> Run()
{
var runCount = 1;
RunSummary summary = new RunSummary();
var test = (ITest) new XunitTest(this.TestCase, this.DisplayName);
var output = string.Empty;
while (runCount <= _runCount)
{
await this.AfterTestCaseStartingAsync();
summary = new RunSummary()
{
Total = 1
};
this.Aggregator = new ExceptionAggregator(Aggregator);
var customerRunner = new XunitCustomRunner(test, this.MessageBus, this.TestClass, this.ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, this.Aggregator, CancellationTokenSource);
output = await customerRunner.RunAsync(summary);
await this.BeforeTestCaseFinishedAsync();
var exception = this.Aggregator.ToException();
if (exception == null)
{
var testResultMessage = (TestResultMessage) new TestPassed(test, summary.Time, output);
if (!this.CancellationTokenSource.IsCancellationRequested && !this.MessageBus.QueueMessage((IMessageSinkMessage)testResultMessage))
{
this.CancellationTokenSource.Cancel();
}
break;
}
if (exception.GetType() == _exceptionToIgnoreTest)
{
SkipReason = exception.Message;
var testResultMessage = (TestResultMessage) new TestSkipped(test, SkipReason);
if (!this.CancellationTokenSource.IsCancellationRequested && !this.MessageBus.QueueMessage((IMessageSinkMessage)testResultMessage))
{
this.CancellationTokenSource.Cancel();
}
break;
}
runCount++;
}
if (this.Aggregator.HasExceptions)
{
var ex = this.Aggregator.ToException();
var testResult = (TestResultMessage) new TestFailed(test, summary.Time, output, ex);
++summary.Failed;
if (!this.CancellationTokenSource.IsCancellationRequested && !this.MessageBus.QueueMessage((IMessageSinkMessage)testResult))
{
this.CancellationTokenSource.Cancel();
}
}
if (!this.MessageBus.QueueMessage((IMessageSinkMessage) new TestCaseFinished((ITestCase)this.TestCase, summary.Time, summary.Total, summary.Failed, summary.Skipped)))
{
this.CancellationTokenSource.Cancel();
}
return(summary);
}
public override async Task <RunSummary> RunAsync(IMessageSink diagnosticMessageSink, IMessageBus messageBus, object[] constructorArguments, ExceptionAggregator aggregator, CancellationTokenSource cancellationTokenSource)
{
var runSummary = new RunSummary();
if (this.SkipReason != null)
{
runSummary.Skipped++;
runSummary.Total++;
if (!messageBus.QueueMessage(new TestSkipped(new XunitTest(this, this.DisplayName), this.SkipReason)))
{
cancellationTokenSource.Cancel();
}
return(runSummary);
}
if (this.parts == null && this.assemblies == null)
{
this.parts = GetNestedTypesRecursively(this.TestMethod.TestClass.Class.ToRuntimeType()).Where(t => (!t.GetTypeInfo().IsAbstract || t.GetTypeInfo().IsSealed) && !t.GetTypeInfo().IsInterface).ToArray();
}
#if DESKTOP
if (this.compositionVersions.HasFlag(CompositionEngines.V1))
{
var runner = new LegacyMefTestCaseRunner(this, "V1", null, constructorArguments, messageBus, aggregator, cancellationTokenSource, CompositionEngines.V1, this.parts, this.assemblies, this.invalidConfiguration);
runSummary.Aggregate(await runner.RunAsync());
}
#endif
if (this.compositionVersions.HasFlag(CompositionEngines.V2))
{
var runner = new LegacyMefTestCaseRunner(this, "V2", null, constructorArguments, messageBus, aggregator, cancellationTokenSource, CompositionEngines.V2, this.parts, this.assemblies, this.invalidConfiguration);
runSummary.Aggregate(await runner.RunAsync());
}
if ((this.compositionVersions & CompositionEngines.V3EnginesMask) == CompositionEngines.Unspecified)
{
if (!this.noCompatGoal)
{
// Call out that we're *not* testing V3 functionality for this test.
if (!messageBus.QueueMessage(new TestSkipped(new XunitTest(this, "V3"), "Test does not include V3 test.")))
{
cancellationTokenSource.Cancel();
}
}
}
else
{
var v3DiscoveryTest = new Mef3DiscoveryTestCaseRunner(this, "V3 composition", null, constructorArguments, messageBus, aggregator, cancellationTokenSource, this.compositionVersions, this.parts ?? new Type[0], this.assemblies ?? ImmutableList <string> .Empty, this.invalidConfiguration);
runSummary.Aggregate(await v3DiscoveryTest.RunAsync());
if (v3DiscoveryTest.Passed && (!this.invalidConfiguration || this.compositionVersions.HasFlag(CompositionEngines.V3AllowConfigurationWithErrors)))
{
foreach (var namedConfiguration in v3DiscoveryTest.ResultingConfigurations)
{
string name = "V3 engine";
if (!string.IsNullOrEmpty(namedConfiguration.Description))
{
name += $"({namedConfiguration.Description})";
}
var runner = new Mef3TestCaseRunner(this, name, null, constructorArguments, messageBus, aggregator, cancellationTokenSource, namedConfiguration.Configuration, this.compositionVersions);
runSummary.Aggregate(await runner.RunAsync());
}
}
}
return(runSummary);
}
public static async Task <RunSummary> RunTestAsync(XunitTestRunner runner,
IMessageBus messageBus,
ExceptionAggregator aggregator,
bool disableRetry)
{
// defense in depth to avoid forever lock
bool acquired = await _semaphore.WaitAsync(TimeSpan.FromHours(2));
try
{
DelayedMessageBus delayedMessageBus = null;
RunSummary summary = null;
if (!acquired)
{
throw new TimeoutException("Wait for thread to run the test timeout!");
}
// First run
if (!disableRetry)
{
// This is really the only tricky bit: we need to capture and delay messages (since those will
// contain run status) until we know we've decided to accept the final result;
delayedMessageBus = new DelayedMessageBus(messageBus);
runner.SetMessageBus(delayedMessageBus);
summary = await RunTestInternalAsync(runner);
// if succeeded
if (summary.Failed == 0 || aggregator.HasExceptions)
{
delayedMessageBus.Flush(false);
return(summary);
}
}
// Final run
runner.SetMessageBus(new KuduTraceMessageBus(messageBus));
summary = await RunTestInternalAsync(runner);
// flush delay messages
if (delayedMessageBus != null)
{
delayedMessageBus.Flush(summary.Failed == 0 && !aggregator.HasExceptions);
}
return(summary);
}
catch (Exception ex)
{
// this is catastrophic
messageBus.QueueMessage(new TestFailed(runner.GetTest(), 0, null, ex));
return(new RunSummary {
Failed = 1, Total = 1
});
}
finally
{
if (acquired)
{
_semaphore.Release();
}
// set to original
runner.SetMessageBus(messageBus);
}
}
private async Task <RunSummary> InvokeStepsAsync(
ICollection <IStepDefinition> backGroundStepDefinitions, ICollection <IStepDefinition> scenarioStepDefinitions)
{
var filters = this.scenarioClass.Assembly.GetCustomAttributes(typeof(Attribute))
.Concat(this.scenarioClass.GetCustomAttributes(typeof(Attribute)))
.Concat(this.scenarioMethod.GetCustomAttributes(typeof(Attribute)))
.OfType <IFilter <IStepDefinition> >();
var stepDefinitions = filters
.Aggregate(
backGroundStepDefinitions.Concat(scenarioStepDefinitions),
(current, filter) => filter.Filter(current))
.ToArray();
var summary = new RunSummary();
string skipReason = null;
var teardowns = new List <Action>();
var stepNumber = 0;
foreach (var stepDefinition in stepDefinitions)
{
stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
stepNumber <= backGroundStepDefinitions.Count,
stepDefinition.Text,
this.scenarioMethodArguments);
var step = new Step(this.scenario, stepDisplayName);
var interceptingBus = new DelegatingMessageBus(
this.messageBus,
message =>
{
if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip)
{
skipReason = string.Format(
CultureInfo.InvariantCulture,
"Failed to execute preceding step: {0}",
step.DisplayName);
}
});
var stepRunner = new StepRunner(
step,
stepDefinition.Body,
interceptingBus,
this.scenarioClass,
this.constructorArguments,
this.scenarioMethod,
this.scenarioMethodArguments,
stepDefinition.SkipReason,
new ExceptionAggregator(this.aggregator),
this.cancellationTokenSource);
summary.Aggregate(await stepRunner.RunAsync());
teardowns.AddRange(stepRunner.Disposables.Select(disposable => (Action)disposable.Dispose)
.Concat(stepDefinition.Teardowns.Where(teardown => teardown != null)).ToArray());
}
if (teardowns.Any())
{
teardowns.Reverse();
var teardownTimer = new ExecutionTimer();
var teardownAggregator = new ExceptionAggregator();
foreach (var teardown in teardowns)
{
teardownTimer.Aggregate(() => teardownAggregator.Run(() => teardown()));
}
summary.Time += teardownTimer.Total;
if (teardownAggregator.HasExceptions)
{
summary.Failed++;
summary.Total++;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
false,
"(Teardown)",
this.scenarioMethodArguments);
this.messageBus.Queue(
new Step(this.scenario, stepDisplayName),
test => new TestFailed(test, teardownTimer.Total, null, teardownAggregator.ToException()),
this.cancellationTokenSource);
}
}
return(summary);
}
async Task DoCorrelation()
{
var outputs = getOutputs();
var usedRoleInstanceNames = new HashSet <string>();
string getUniqueRoleInstanceName(ILogSource logSource)
{
for (int tryCount = 0; ; ++tryCount)
{
var ret = string.Format(
tryCount == 0 ? "{0}" : "{0} ({1})",
logSource.GetShortDisplayNameWithAnnotation(),
tryCount
);
if (usedRoleInstanceNames.Add(ret))
{
return(ret);
}
}
}
NodeId makeNodeId(ILogSource logSource) =>
new NodeId(logSource.Provider.Factory.ToString(), getUniqueRoleInstanceName(logSource));
var allLogs =
outputs
.Select(output => output.OutputData)
.OfType <ICorrelatorOutput>()
.Select(data => new NodeInfo(data.LogSource, makeNodeId(data.LogSource), data.RotatedLogPartToken, data.Events, data.SameNodeDetectionToken))
.ToArray();
var fixedConstraints =
allLogs
.GroupBy(l => l.SameNodeDetectionToken, new SameNodeEqualityComparer())
.SelectMany(group => LinqUtils.ZipWithNext(group).Select(pair => new FixedConstraint()
{
Node1 = pair.Key.NodeId,
Node2 = pair.Value.NodeId,
Value = pair.Value.SameNodeDetectionToken.DetectSameNode(pair.Key.SameNodeDetectionToken).TimeDiff
}))
.ToList();
ICorrelator correlator = new Correlator(new InternodeMessagesDetector(), solverFactory());
var allowInstacesMergingForRoles = new HashSet <string>();
var correlatorSolution = await correlator.Correlate(
allLogs.ToDictionary(i => i.NodeId, i => i.Messages),
fixedConstraints,
allowInstacesMergingForRoles
);
var nodeIdToLogSources =
(from l in allLogs
from ls in l.LogSources
select new { L = l.NodeId, Ls = ls })
.ToLookup(i => i.L, i => i.Ls);
var grouppedLogsReport = new StringBuilder();
foreach (var group in allLogs.Where(g => g.LogSources.Length > 1))
{
if (grouppedLogsReport.Length == 0)
{
grouppedLogsReport.AppendLine();
grouppedLogsReport.AppendLine("Groupped logs info:");
}
grouppedLogsReport.AppendLine(string.Format(
" - {0} were groupped to represent node {1}",
string.Join(", ", group.LogSources.Select(ls => '\"' + ls.GetShortDisplayNameWithAnnotation() + '\"')),
group.NodeId));
}
var timeOffsets =
(from ns in correlatorSolution.NodeSolutions
from ls in nodeIdToLogSources[ns.Key]
select new { Sln = ns.Value, Ls = ls })
.ToImmutableDictionary(i => i.Ls, i => i.Sln);
await modelThreadSync.Invoke(() =>
{
foreach (var ls in logSourcesManager.Items)
{
if (timeOffsets.TryGetValue(ls, out var sln))
{
ITimeOffsetsBuilder builder = logSourcesManager.CreateTimeOffsetsBuilder();
builder.SetBaseOffset(sln.BaseDelta);
if (sln.TimeDeltas != null)
{
foreach (var d in sln.TimeDeltas)
{
builder.AddOffset(d.At, d.Delta);
}
}
ls.TimeOffsets = builder.ToTimeOffsets();
}
}
});
logConnectionIdToLastRunResult = timeOffsets.ToImmutableDictionary(
to => to.Key.GetSafeConnectionId(),
to => new CorrelatorRunResult(to.Value, GetCorrelatableLogsConnectionIds(outputs))
);
lastRunSummary = new RunSummary {
Report = correlatorSolution.CorrelationLog + grouppedLogsReport
};
changeNotification.Post();
}
static CorrelationStateSummary GetCorrelatorStateSummary(
ImmutableArray <LogSourcePostprocessorState> correlationOutputs,
ImmutableDictionary <string, CorrelatorRunResult> lastResult,
RunSummary lastRunSummary,
int _timeShiftsRevision
)
{
if (correlationOutputs.Length < 2)
{
return(new CorrelationStateSummary {
Status = CorrelationStateSummary.StatusCode.PostprocessingUnavailable
});
}
if (lastRunSummary?.IsFailure == true)
{
return(new CorrelationStateSummary {
Status = CorrelationStateSummary.StatusCode.ProcessingFailed, Report = lastRunSummary.Report
});
}
var correlatableLogsIds = GetCorrelatableLogsConnectionIds(correlationOutputs);
int numMissingOutput = 0;
int numProgressing = 0;
int numFailed = 0;
int numCorrelationContextMismatches = 0;
int numCorrelationResultMismatches = 0;
double?progress = null;
foreach (var i in correlationOutputs)
{
if (i.OutputStatus == LogSourcePostprocessorState.Status.InProgress ||
i.OutputStatus == LogSourcePostprocessorState.Status.Loading)
{
numProgressing++;
if (progress == null && i.Progress != null)
{
progress = i.Progress;
}
}
lastResult.TryGetValue(i.LogSource.GetSafeConnectionId(), out var typedOutput);
if (typedOutput == null)
{
++numMissingOutput;
}
else
{
if (!typedOutput.CorrelatedLogsConnectionIds.IsSupersetOf(correlatableLogsIds))
{
++numCorrelationContextMismatches;
}
var actualOffsets = i.LogSource.IsDisposed ? TimeOffsets.Empty : i.LogSource.Provider.TimeOffsets;
if (typedOutput.Solution.BaseDelta != actualOffsets.BaseOffset)
{
++numCorrelationResultMismatches;
}
}
if (i.OutputStatus == LogSourcePostprocessorState.Status.Failed)
{
++numFailed;
}
}
if (numProgressing != 0)
{
return(new CorrelationStateSummary()
{
Status = CorrelationStateSummary.StatusCode.ProcessingInProgress,
Progress = progress
});
}
string report = lastRunSummary?.Report;
if (numFailed != 0)
{
IPostprocessorRunSummary summaryWithError = correlationOutputs
.Select(output => output.LastRunSummary)
.OfType <IPostprocessorRunSummary>()
.Where(summary => summary.HasErrors)
.FirstOrDefault();
if (summaryWithError != null)
{
return(new CorrelationStateSummary()
{
Status = CorrelationStateSummary.StatusCode.ProcessingFailed,
Report = summaryWithError.Report
});
}
return(new CorrelationStateSummary()
{
Status = CorrelationStateSummary.StatusCode.ProcessingFailed,
Report = report
});
}
if (numMissingOutput != 0 || numCorrelationContextMismatches != 0 || numCorrelationResultMismatches != 0)
{
return(new CorrelationStateSummary()
{
Status = CorrelationStateSummary.StatusCode.NeedsProcessing
});
}
return(new CorrelationStateSummary()
{
Status = CorrelationStateSummary.StatusCode.Processed,
Report = report
});
}
/// <inheritdoc/>
public void Run(IEnumerable<ITestCase> testCases, IMessageSink messageSink)
{
var results = new RunSummary();
var environment = String.Format("{0}-bit .NET {1}", IntPtr.Size * 8, Environment.Version);
// TODO: Contract for Run() states that null "testCases" means "run everything".
if (messageSink.OnMessage(new TestAssemblyStarting(assemblyFileName, configFileName, DateTime.Now, environment, TestFrameworkDisplayName)))
foreach (var testCollectionGroup in testCases.Cast<Xunit1TestCase>().GroupBy(tc => tc.TestCollection))
{
var collectionResults = RunTestCollection(testCollectionGroup.Key, testCollectionGroup, messageSink);
results.Aggregate(collectionResults);
if (!collectionResults.Continue)
break;
}
messageSink.OnMessage(new TestAssemblyFinished(new Xunit1AssemblyInfo(assemblyFileName), results.Time, results.Total, results.Failed, results.Skipped));
}
/// <summary>
/// Runs the tests in the test method.
/// </summary>
/// <param name="ctxt">The context that describes the current test method</param>
/// <returns>Returns summary information about the tests that were run.</returns>
protected async ValueTask <RunSummary> RunAsync(TContext ctxt)
{
await ctxt.InitializeAsync();
try
{
SetTestContext(ctxt, TestEngineStatus.Initializing);
var methodSummary = new RunSummary();
var testCollection = ctxt.TestCases.First().TestCollection;
var testAssemblyUniqueID = testCollection.TestAssembly.UniqueID;
var testCollectionUniqueID = testCollection.UniqueID;
var testClassUniqueID = ctxt.TestClass.UniqueID;
var testMethodUniqueID = ctxt.TestMethod.UniqueID;
var methodStarting = new _TestMethodStarting
{
AssemblyUniqueID = testAssemblyUniqueID,
TestClassUniqueID = testClassUniqueID,
TestCollectionUniqueID = testCollectionUniqueID,
TestMethod = ctxt.TestMethod.Method.Name,
TestMethodUniqueID = testMethodUniqueID
};
if (!ctxt.MessageBus.QueueMessage(methodStarting))
{
ctxt.CancellationTokenSource.Cancel();
return(methodSummary);
}
try
{
await AfterTestMethodStarting(ctxt);
SetTestContext(ctxt, TestEngineStatus.Running);
methodSummary = await RunTestCasesAsync(ctxt);
SetTestContext(ctxt, TestEngineStatus.CleaningUp);
ctxt.Aggregator.Clear();
await BeforeTestMethodFinished(ctxt);
if (ctxt.Aggregator.HasExceptions)
{
var methodCleanupFailure = _TestMethodCleanupFailure.FromException(ctxt.Aggregator.ToException() !, testAssemblyUniqueID, testCollectionUniqueID, testClassUniqueID, testMethodUniqueID);
if (!ctxt.MessageBus.QueueMessage(methodCleanupFailure))
{
ctxt.CancellationTokenSource.Cancel();
}
}
return(methodSummary);
}
finally
{
var testMethodFinished = new _TestMethodFinished
{
AssemblyUniqueID = testAssemblyUniqueID,
ExecutionTime = methodSummary.Time,
TestClassUniqueID = testClassUniqueID,
TestCollectionUniqueID = testCollectionUniqueID,
TestMethodUniqueID = testMethodUniqueID,
TestsFailed = methodSummary.Failed,
TestsRun = methodSummary.Total,
TestsSkipped = methodSummary.Skipped
};
if (!ctxt.MessageBus.QueueMessage(testMethodFinished))
{
ctxt.CancellationTokenSource.Cancel();
}
}
}
finally
{
await ctxt.DisposeAsync();
}
}
private static void GenerateProcessingErrorsSection(StringBuilder sb, RunSummary runSummary)
{
sb.AppendLine(GenerateTitle("Processing errors"));
sb.AppendLine(runSummary.ProcessingErrorsReport());
}
public Task DisposeAsync(RunSummary result, IMessageBus messageBus) => ContainerHost.Decrement();
private static void GenerateWorkbookGeneratorResultsSection(StringBuilder sb, RunSummary runSummary)
{
sb.AppendLine(GenerateTitle("Generated workbooks"));
var workbookTemplates = runSummary.WorkbookGeneratorResults.WorkbookTemplates ?? Enumerable.Empty <PackagedWorkbookTemplateInfo>();
var allAvailableTemplates = workbookTemplates.Select(template => template.Name).OrderBy(name => name);
sb.AppendLine($"Workbook templates available: {string.Join("; ", allAvailableTemplates)}");
var completedWorkbooks = runSummary.WorkbookGeneratorResults.CompletedWorkbooks ?? Enumerable.Empty <CompletedWorkbookInfo>();
var successfullyGenerated = completedWorkbooks.Where(cwi => cwi.GeneratedSuccessfully).ToList();
var workbooksWithData = successfullyGenerated
.Where(cwi => cwi.HasAnyData)
.Select(cwi => cwi.OriginalWorkbookName)
.OrderBy(workbookName => workbookName)
.ToList();
if (workbooksWithData.Count > 0)
{
sb.AppendLine($"Successfully generated workbooks with data: {GenerateHorizontalList(workbooksWithData)}");
}
var workbooksWithoutData = successfullyGenerated
.Where(cwi => !cwi.HasAnyData)
.Select(cwi => cwi.OriginalWorkbookName)
.OrderBy(workbookName => workbookName)
.ToList();
if (workbooksWithoutData.Count > 0)
{
sb.AppendLine($"Empty workbooks: {GenerateHorizontalList(workbooksWithoutData)}");
}
if (runSummary.HasWorkbookGeneratorErrors())
{
sb.AppendLine(runSummary.WorkbookGeneratorErrorsReport());
}
}
/// <summary>
/// Returns true if PipelineRunImpl instances are equal
/// </summary>
/// <param name="other">Instance of PipelineRunImpl to be compared</param>
/// <returns>Boolean</returns>
public bool Equals(PipelineRunImpl other)
{
if (other is null)
{
return(false);
}
if (ReferenceEquals(this, other))
{
return(true);
}
return
((
Class == other.Class ||
Class != null &&
Class.Equals(other.Class)
) &&
(
Links == other.Links ||
Links != null &&
Links.Equals(other.Links)
) &&
(
DurationInMillis == other.DurationInMillis ||
DurationInMillis.Equals(other.DurationInMillis)
) &&
(
EnQueueTime == other.EnQueueTime ||
EnQueueTime != null &&
EnQueueTime.Equals(other.EnQueueTime)
) &&
(
EndTime == other.EndTime ||
EndTime != null &&
EndTime.Equals(other.EndTime)
) &&
(
EstimatedDurationInMillis == other.EstimatedDurationInMillis ||
EstimatedDurationInMillis.Equals(other.EstimatedDurationInMillis)
) &&
(
Id == other.Id ||
Id != null &&
Id.Equals(other.Id)
) &&
(
Organization == other.Organization ||
Organization != null &&
Organization.Equals(other.Organization)
) &&
(
Pipeline == other.Pipeline ||
Pipeline != null &&
Pipeline.Equals(other.Pipeline)
) &&
(
Result == other.Result ||
Result != null &&
Result.Equals(other.Result)
) &&
(
RunSummary == other.RunSummary ||
RunSummary != null &&
RunSummary.Equals(other.RunSummary)
) &&
(
StartTime == other.StartTime ||
StartTime != null &&
StartTime.Equals(other.StartTime)
) &&
(
State == other.State ||
State != null &&
State.Equals(other.State)
) &&
(
Type == other.Type ||
Type != null &&
Type.Equals(other.Type)
) &&
(
CommitId == other.CommitId ||
CommitId != null &&
CommitId.Equals(other.CommitId)
));
}
public static bool LogReadingCompletedSuccessfully(this RunSummary runSummary)
{
return(runSummary?.ProcessLogSetResult != null &&
runSummary.ProcessLogSetResult.IsSuccessful);
}
private async Task <RunSummary> InvokeStepsAsync(
ICollection <IStepDefinition> backGroundStepDefinitions, ICollection <IStepDefinition> scenarioStepDefinitions)
{
var scenarioTypeInfo = this.scenarioClass.GetTypeInfo();
var filters = scenarioTypeInfo.Assembly.GetCustomAttributes(typeof(Attribute))
.Concat(scenarioTypeInfo.GetCustomAttributes(typeof(Attribute)))
.Concat(this.scenarioMethod.GetCustomAttributes(typeof(Attribute)))
.OfType <IFilter <IStepDefinition> >();
var summary = new RunSummary();
string skipReason = null;
var scenarioTeardowns = new List <Tuple <StepContext, Func <IStepContext, Task> > >();
var stepNumber = 0;
foreach (var stepDefinition in filters.Aggregate(
backGroundStepDefinitions.Concat(scenarioStepDefinitions),
(current, filter) => filter.Filter(current)))
{
stepDefinition.SkipReason = stepDefinition.SkipReason ?? skipReason;
var stepDisplayName = GetStepDisplayName(
this.scenario.DisplayName,
++stepNumber,
stepDefinition.DisplayTextFunc?.Invoke(stepDefinition.Text, stepNumber <= backGroundStepDefinitions.Count));
var step = new StepTest(this.scenario, stepDisplayName);
var interceptingBus = new DelegatingMessageBus(
this.messageBus,
message =>
{
if (message is ITestFailed && stepDefinition.FailureBehavior == RemainingSteps.Skip)
{
skipReason = $"Failed to execute preceding step: {step.DisplayName}";
}
});
var stepContext = new StepContext(step);
var stepRunner = new StepTestRunner(
stepContext,
stepDefinition.Body,
step,
interceptingBus,
this.scenarioClass,
this.constructorArguments,
this.scenarioMethod,
this.scenarioMethodArguments,
stepDefinition.SkipReason,
new BeforeAfterTestAttribute[0],
new ExceptionAggregator(this.aggregator),
this.cancellationTokenSource);
summary.Aggregate(await stepRunner.RunAsync());
var stepTeardowns = stepContext.Disposables
.Where(disposable => disposable != null)
.Select((Func <IDisposable, Func <IStepContext, Task> >)(disposable =>
context =>
{
disposable.Dispose();
return(Task.FromResult(0));
}))
.Concat(stepDefinition.Teardowns)
.Where(teardown => teardown != null)
.Select(teardown => Tuple.Create(stepContext, teardown));
scenarioTeardowns.AddRange(stepTeardowns);
}
if (scenarioTeardowns.Any())
{
scenarioTeardowns.Reverse();
var teardownTimer = new ExecutionTimer();
var teardownAggregator = new ExceptionAggregator();
foreach (var teardown in scenarioTeardowns)
{
await Invoker.Invoke(() => teardown.Item2(teardown.Item1), teardownAggregator, teardownTimer);
}
summary.Time += teardownTimer.Total;
if (teardownAggregator.HasExceptions)
{
summary.Failed++;
summary.Total++;
var stepDisplayName = GetStepDisplayName(this.scenario.DisplayName, ++stepNumber, "(Teardown)");
this.messageBus.Queue(
new StepTest(this.scenario, stepDisplayName),
test => new TestFailed(test, teardownTimer.Total, null, teardownAggregator.ToException()),
this.cancellationTokenSource);
}
}
return(summary);
}
public static bool HasProcessingErrors(this RunSummary runSummary)
{
return(runSummary?.ProcessingNotificationsCollector != null &&
runSummary.ProcessingNotificationsCollector.TotalErrorsReported > 0);
}
public static TestableTestCaseRunner Create(IMessageBus messageBus, ITestCase testCase = null, RunSummary result = null, Exception aggregatorSeedException = null)
{
var aggregator = new ExceptionAggregator();
if (aggregatorSeedException != null)
{
aggregator.Add(aggregatorSeedException);
}
return(new TestableTestCaseRunner(
testCase ?? Mocks.TestCase <Object>("ToString"),
messageBus,
aggregator,
new CancellationTokenSource(),
result ?? new RunSummary()
));
}
public static bool HasWorkbookGeneratorErrors(this RunSummary runSummary)
{
return(runSummary?.WorkbookGeneratorResults?.CompletedWorkbooks != null &&
runSummary.WorkbookGeneratorResults.CompletedWorkbooks.Any(result => !result.GeneratedSuccessfully));
}
protected override async Task <RunSummary> RunTestAsync()
{
var scenarioFactTestCase = (ScenarioFactTestCase)TestCase;
var test = CreateTest(TestCase, DisplayName);
var aggregatedResult = new RunSummary();
// Theories are called with required arguments. Keep track of what arguments we already tested so that we can skip those accordingly
var testedArguments = new HashSet <object>();
// Each time we find a new theory argument, we will want to restart our Test so that we can collect subsequent test cases
bool pendingRestart;
do
{
// Safeguarding against abuse
if (testedArguments.Count >= scenarioFactTestCase.TheoryTestCaseLimit)
{
pendingRestart = false;
MessageBus.QueueMessage(new TestSkipped(test, "Theory tests are capped to prevent infinite loops. You can configure a different limit by setting TheoryTestCaseLimit on the Scenario attribute"));
aggregatedResult.Aggregate(new RunSummary
{
Skipped = 1,
Total = 1
});
}
else
{
var bufferedMessageBus = new BufferedMessageBus(MessageBus);
var stopwatch = Stopwatch.StartNew();
var skipAdditionalTests = false;
var testRecorded = false;
pendingRestart = false; // By default we dont expect a new restart
object? capturedArgument = null;
ScenarioContext scenarioContext = null;
scenarioContext = new ScenarioContext(scenarioFactTestCase.FactName, async(ScenarioTestCaseDescriptor descriptor) =>
{
// If we're hitting our target test
if (descriptor.Name == scenarioFactTestCase.FactName)
{
testRecorded = true;
if (skipAdditionalTests)
{
pendingRestart = true; // when we discovered more tests after a test completed, allow us to restart
scenarioContext.EndScenarioConditionally();
return;
}
if (descriptor.Argument is not null)
{
// If we've already received this test case, don't run it again
if (testedArguments.Contains(descriptor.Argument))
{
return;
}
testedArguments.Add(descriptor.Argument);
capturedArgument = descriptor.Argument;
}
// At this stage we found our first valid test case, any subsequent test case should issue a restart instead
skipAdditionalTests = true;
try
{
await descriptor.Invocation();
}
catch (Exception)
{
// If we caught an exception but we're in a theory, we will want to try for additional test cases
if (descriptor.Argument is not null)
{
pendingRestart = true;
}
throw;
}
finally
{
scenarioContext.IsTargetConclusive = true;
}
}
else
{
// We may be hitting a shared fact, those need to be invoked as well but not recorded as our primary target
if (!scenarioFactTestCase.RunInIsolation || descriptor.Flags.HasFlag(ScenarioTestCaseFlags.Shared))
{
await descriptor.Invocation();
}
}
});
scenarioContext.AutoAbort = scenarioFactTestCase.ExecutionPolicy is ScenarioTestExecutionPolicy.EndAfterConclusion;
TestMethodArguments = new object[] { scenarioContext };
RunSummary result;
result = await CreateTestRunner(test, bufferedMessageBus, TestClass, ConstructorArguments, TestMethod, TestMethodArguments, SkipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource).RunAsync();
aggregatedResult.Aggregate(result);
stopwatch.Stop();
var testInvocationTest = capturedArgument switch
{
null => CreateTest(TestCase, DisplayName),
not null => CreateTest(TestCase, $"{DisplayName} ({capturedArgument})")
};
var bufferedMessages = bufferedMessageBus.QueuedMessages;
// We should have expected at least one test run. We probably returned before our target test was able to run
if (!testRecorded && result.Failed == 0)
{
bufferedMessageBus.QueueMessage(new TestSkipped(test, scenarioContext.SkippedReason ?? "No applicable tests were able to run"));
result = new RunSummary {
Skipped = 1, Total = 1
};
}
// If we skipped this test, make sure that this is reported accordingly
if (scenarioContext.Skipped && !bufferedMessages.OfType <TestSkipped>().Any())
{
bufferedMessages = bufferedMessages.Concat(new[] { new TestSkipped(testInvocationTest, scenarioContext.SkippedReason) });
}
// If we have indeed skipped this test, make sure that we're not reporting it as passed or failed
if (bufferedMessages.OfType <TestSkipped>().Any())
{
bufferedMessages = bufferedMessages.Where(x => x is not TestPassed and not TestFailed);
}
// If we have a failure in post conditions, don't mark this test case as passed
if (bufferedMessages.OfType <TestFailed>().Any())
{
bufferedMessages = bufferedMessages.Where(x => x is not TestPassed);
}
var output = string.Join("", bufferedMessages
.OfType <ITestOutput>()
.Select(x => x.Output));
var duration = (decimal)stopwatch.Elapsed.TotalSeconds;
foreach (var queuedMessage in bufferedMessages)
{
var transformedMessage = queuedMessage switch
{
TestStarting testStarting => new TestStarting(testInvocationTest),
TestSkipped testSkipped => new TestSkipped(testInvocationTest, testSkipped.Reason),
TestPassed testPassed => new TestPassed(testInvocationTest, duration, output),
TestFailed testFailed => new TestFailed(testInvocationTest, duration, output, testFailed.ExceptionTypes, testFailed.Messages, testFailed.StackTraces, testFailed.ExceptionParentIndices),
TestFinished testFinished => new TestFinished(testInvocationTest, duration, output),
_ => queuedMessage
};
if (!MessageBus.QueueMessage(transformedMessage))
{
return(aggregatedResult);
}
}
}
}while (pendingRestart);
return(aggregatedResult);
}
}
}
public static string LogSetProcessingStatusReport(this RunSummary runSummary)
{
return(runSummary.LogReadingCompletedSuccessfully()
? "LogShark successfully processed all logs"
: $"Error occurred while processing logs: {runSummary.ProcessLogSetResult.ErrorMessage}");
}
private async Task <RunSummary> InvokeScenarioMethodAsync(object scenarioClassInstance)
{
var backgroundSteps = new List <IStepDefinition>();
var scenarioSteps = new List <IStepDefinition>();
var tearDownSteps = new List <IStepDefinition>();
await this._aggregator.RunAsync(async() =>
{
// We leverage this for both Background and TearDown methods.
async Task OnDiscoverSupportMethodsAsync <TAttribute>(
IScenario scenario, ExecutionTimer timer)
where TAttribute : SupportMethodAttribute
{
var methods = scenario.TestCase.TestMethod.TestClass.Class
.GetMethods(false)
.Where(candidate => candidate.GetCustomAttributes(typeof(TAttribute)).Any())
.Select(method => method.ToRuntimeMethod())
// #2 MWP 2020-07-01 12:28:03 PM: the rubber meeting the road here.
.OrderBy(method => method, new MethodInfoComparer()).ToList();
// Thus ensuring correct front to back Background, reverse TearDown.
if (typeof(TAttribute) == typeof(TearDownAttribute))
{
methods.Reverse();
}
foreach (var method in methods)
{
/* #4 MWP 2020-07-09 05:47:25 PM / We support seeding default values into
* Background methods with parameters. However, this is the extent of what
* we can accomplish there. It does not make any sense to cross any Theory
* dataRow bridges for Background methods. */
var argTypes = method.GetParameters().Select(param => param.ParameterType).ToArray();
var args = argTypes.Select(paramType => paramType.GetDefault()).ToArray();
var convertedArgs = Reflector.ConvertArguments(args, argTypes);
convertedArgs = convertedArgs.Any() ? convertedArgs : null;
await timer.AggregateAsync(() => method.InvokeAsync(scenarioClassInstance, convertedArgs));
}
}
using (CurrentThread.EnterStepDefinitionContext())
{
await OnDiscoverSupportMethodsAsync <BackgroundAttribute>(this._scenario, this._timer);
backgroundSteps.AddRange(CurrentThread.StepDefinitions);
}
using (CurrentThread.EnterStepDefinitionContext())
{
await this._timer.AggregateAsync(() => this._scenarioMethod.InvokeAsync(scenarioClassInstance, this._scenarioMethodArguments));
scenarioSteps.AddRange(CurrentThread.StepDefinitions);
}
// MWP Sun, 06 Dec 2020 11:00:56 AM / Rinse and repeat Background, except for TearDown...
using (CurrentThread.EnterStepDefinitionContext())
{
await OnDiscoverSupportMethodsAsync <TearDownAttribute>(this._scenario, this._timer);
tearDownSteps.AddRange(CurrentThread.StepDefinitions);
}
});
var runSummary = new RunSummary {
Time = this._timer.Total
};
if (!this._aggregator.HasExceptions)
{
// Assumes Scenario for StepDefinitionType.
backgroundSteps.ForEach(x => x.StepDefinitionType = StepType.Background);
tearDownSteps.ForEach(x => x.StepDefinitionType = StepType.TearDown);
runSummary.Aggregate(await this.InvokeStepsAsync(backgroundSteps, scenarioSteps, tearDownSteps));
}
return(runSummary);
}
private async Task ExecuteTestMethod(MethodInfo runtimeMethod, RunSummary runSummary, IEnumerable <WebDriverAttribute> driverAttributes, object[] dataRow)
{
foreach (var driverAttribute in driverAttributes)
{
foreach (var driver in driverAttribute.GetDrivers(runtimeMethod))
{
Fixture newFixture = null;
object initializerReturn = null;
ITypeInfo[] resolvedTypes = null;
var methodToRun = runtimeMethod;
if (methodToRun.IsGenericMethodDefinition)
{
resolvedTypes = TestCase.TestMethod.Method.ResolveGenericTypes(dataRow);
methodToRun = methodToRun.MakeGenericMethod(resolvedTypes.Select(t => ((IReflectionTypeInfo)t).Type).ToArray());
}
List <object> parameterList = new List <object>();
var parameters = methodToRun.GetParameters().ToArray();
try
{
newFixture = FixtureCreationAttribute.GetNewFixture(driver, runtimeMethod);
var initializeDataAttributes = ReflectionHelper.GetAttributes <FixtureInitializationAttribute>(runtimeMethod);
foreach (var initializeDataAttribute in initializeDataAttributes)
{
if (initializeDataAttribute is IMethodInfoAware)
{
#if DNX
var property = initializeDataAttribute.GetType().GetRuntimeProperty("Method");
property.SetValue(initializeDataAttribute, runtimeMethod);
#else
((IMethodInfoAware)initializeDataAttribute).Method = runtimeMethod;
#endif
}
initializeDataAttribute.Initialize(newFixture.Data);
}
var initializeAttribute = ReflectionHelper.GetAttribute <IFixtureInitializationAttribute>(runtimeMethod);
if (initializeAttribute != null)
{
initializerReturn = initializeAttribute.Initialize(runtimeMethod, newFixture);
}
int dataRowIndex = 0;
for (int i = 0; i < parameters.Length; i++)
{
var parameter = parameters[i];
var attributes = parameter.GetCustomAttributes(true);
if (parameter.ParameterType == typeof(IWebDriver))
{
parameterList.Add(driver);
}
else if (parameter.ParameterType == typeof(Fixture))
{
parameterList.Add(newFixture);
}
else if (attributes.Any(a => a is GenerateAttribute))
{
var generateAttribute = (GenerateAttribute)attributes.First(a => a is GenerateAttribute);
InitializeCustomAttribute(generateAttribute, runtimeMethod, parameter);
var constraintName = generateAttribute.ConstraintName ?? parameter.Name;
var min = generateAttribute.Min;
var max = generateAttribute.Max;
var value = newFixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max });
parameterList.Add(value);
}
else if (attributes.Any(a => a is LocateAttribute))
{
var locateAttribute = (LocateAttribute)attributes.First(a => a is LocateAttribute);
InitializeCustomAttribute(locateAttribute, runtimeMethod, parameter);
var value = locateAttribute.Value;
if (value == null)
{
value = newFixture.Data.Generate(new SimpleFixture.DataRequest(null,
newFixture.Data,
parameter.ParameterType,
parameter.Name,
false,
null,
null));
}
parameterList.Add(value);
}
else if (attributes.Any(a => a is FreezeAttribute))
{
var freeze = (FreezeAttribute)attributes.FirstOrDefault(a => a is FreezeAttribute);
InitializeCustomAttribute(freeze, runtimeMethod, parameter);
var value = freeze.Value;
if (value == null)
{
var constraintName = freeze.ConstraintName ?? parameter.Name;
var min = freeze.Min;
var max = freeze.Max;
value = newFixture.Data.Generate(parameter.ParameterType, constraintName, new { min, max });
}
parameterList.Add(value);
object lastObject = parameterList.Last();
var closedFreezeMethod =
FreezeMethod.MakeGenericMethod(lastObject.GetType());
closedFreezeMethod.Invoke(null, new object[] { newFixture.Data, value, freeze.For });
}
else if (initializerReturn != null && parameter.ParameterType == initializerReturn.GetType())
{
parameterList.Add(initializerReturn);
initializerReturn = null;
}
else if (dataRowIndex < dataRow.Length)
{
var dataValue = dataRow[dataRowIndex];
dataRowIndex++;
parameterList.Add(dataValue);
}
else
{
var value = newFixture.Data.Generate(parameter.ParameterType, parameter.Name);
parameterList.Add(value);
}
}
}
catch (Exception exp)
{
Aggregator.Add(exp);
}
var convertedDataRow = Reflector.ConvertArguments(parameterList.ToArray(), parameters.Select(p => p.ParameterType).ToArray());
var theoryDisplayName =
TestCase.TestMethod.Method.GetDisplayNameWithArguments(DisplayName + " " + GetDriverName(driver),
dataRow,
resolvedTypes);
var test = new XunitTest(TestCase, theoryDisplayName);
var skipReason = SkipReason;
XunitTestRunner testRunner = CreateTestRunner(test, MessageBus, TestClass, ConstructorArguments, methodToRun, convertedDataRow, skipReason, BeforeAfterAttributes, Aggregator, CancellationTokenSource);
runSummary.Aggregate(await testRunner.RunAsync());
var timer = new ExecutionTimer();
timer.Aggregate(() => DisposeOfData(driverAttribute, driver, newFixture, dataRow));
runSummary.Time += timer.Total;
}
}
}
RunSummary RunTestCollection(ITestCollection testCollection, IEnumerable<Xunit1TestCase> testCases, IMessageSink messageSink)
{
var results = new RunSummary();
results.Continue = messageSink.OnMessage(new TestCollectionStarting(testCollection));
if (results.Continue)
foreach (var testClassGroup in testCases.GroupBy(tc => tc.Class.Name))
{
var classResults = RunTestClass(testCollection, testClassGroup.Key, testClassGroup.ToList(), messageSink);
results.Aggregate(classResults);
if (!classResults.Continue)
break;
}
results.Continue = messageSink.OnMessage(new TestCollectionFinished(testCollection, results.Time, results.Total, results.Failed, results.Skipped)) && results.Continue;
return results;
}
public async Task <RunSummary> RunScenarioAsync()
{
var runSummary = new RunSummary {
Total = 1
};
var output = string.Empty;
if (!MessageBus.QueueMessage(new TestStarting(Test)))
{
CancellationTokenSource.Cancel();
}
else
{
AfterTestStarting();
if (!string.IsNullOrEmpty(SkipReason))
{
runSummary.Skipped++;
if (!MessageBus.QueueMessage(new TestSkipped(Test, SkipReason)))
{
CancellationTokenSource.Cancel();
}
}
else
{
var aggregator = new ExceptionAggregator(Aggregator);
if (!aggregator.HasExceptions)
{
var tuple = await aggregator.RunAsync(() => InvokeTestAsync(aggregator));
runSummary.Time = tuple.Item1;
output = tuple.Item2;
}
var exception = aggregator.ToException();
TestResultMessage testResult;
if (exception == null)
{
testResult = new TestPassed(Test, runSummary.Time, output);
}
else if (exception is IgnoreException)
{
testResult = new TestSkipped(Test, exception.Message);
runSummary.Skipped++;
}
else
{
testResult = new TestFailed(Test, runSummary.Time, output, exception);
runSummary.Failed++;
}
if (!CancellationTokenSource.IsCancellationRequested)
{
if (!MessageBus.QueueMessage(testResult))
{
CancellationTokenSource.Cancel();
}
}
}
Aggregator.Clear();
BeforeTestFinished();
if (Aggregator.HasExceptions)
{
if (!MessageBus.QueueMessage(new TestCleanupFailure(Test, Aggregator.ToException())))
{
CancellationTokenSource.Cancel();
}
}
}
if (!MessageBus.QueueMessage(new TestFinished(Test, runSummary.Time, output)))
{
CancellationTokenSource.Cancel();
}
return(runSummary);
}
public Task DisposeAsync(RunSummary result, IMessageBus messageBus)
{
return(Task.CompletedTask);
}
public virtual Task DisposeAsync(RunSummary result, IMessageBus messageBus)
{
Dispose(true);
return(ContainerHost.Decrement());
}