protected override async Task<RunSummary> RunTestAsync()
{
var runSummary = new BenchmarkRunSummary
{
TestClassFullName = TestCase.TestMethod.TestClass.Class.Name,
TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
TestMethod = TestCase.TestMethodName,
Variation = TestCase.Variation,
ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
RunStarted = DateTime.UtcNow,
MachineName = _machineName,
Framework = TestCase.Framework ?? _framework,
Architecture = IntPtr.Size > 4 ? "x64" : "x86",
WarmupIterations = TestCase.WarmupIterations,
Iterations = TestCase.Iterations,
CustomData = BenchmarkConfig.Instance.CustomData
};
for (var i = 0; i < TestCase.WarmupIterations; i++)
{
var runner = CreateRunner(i + 1, TestCase, warmup: true);
runSummary.Aggregate(await runner.RunAsync());
}
for (var i = 0; i < TestCase.Iterations; i++)
{
TestCase.MetricCollector.Reset();
var runner = CreateRunner(i + 1, TestCase, warmup: false);
var iterationSummary = new BenchmarkIterationSummary();
iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
runSummary.Aggregate(iterationSummary);
}
if (runSummary.Failed != 0)
{
_diagnosticMessageSink.OnMessage(
new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed."));
}
else
{
runSummary.PopulateMetrics();
_diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString()));
//foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
//{
// try
// {
// new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
// }
// catch (Exception ex)
// {
// _diagnosticMessageSink.OnMessage(
// new XunitDiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex}"));
// throw;
// }
//}
}
return runSummary;
}
protected override async Task <RunSummary> RunTestAsync()
{
var runSummary = new BenchmarkRunSummary
{
TestClassFullName = TestCase.TestMethod.TestClass.Class.Name,
TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
TestMethod = TestCase.TestMethodName,
Variation = TestCase.Variation,
ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
RunStarted = DateTime.UtcNow,
MachineName = _machineName,
Framework = TestCase.Framework ?? _framework,
Architecture = IntPtr.Size > 4 ? "x64" : "x86",
WarmupIterations = TestCase.WarmupIterations,
Iterations = TestCase.Iterations,
CustomData = BenchmarkConfig.Instance.CustomData
};
for (var i = 0; i < TestCase.WarmupIterations; i++)
{
var runner = CreateRunner(i + 1, TestCase, warmup: true);
runSummary.Aggregate(await runner.RunAsync());
}
for (var i = 0; i < TestCase.Iterations; i++)
{
TestCase.MetricCollector.Reset();
var runner = CreateRunner(i + 1, TestCase, warmup: false);
var iterationSummary = new BenchmarkIterationSummary();
iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
runSummary.Aggregate(iterationSummary);
}
if (runSummary.Failed != 0)
{
_diagnosticMessageSink.OnMessage(
new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed."));
}
else
{
runSummary.PopulateMetrics();
_diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString()));
foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
{
try
{
new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
}
catch (Exception ex)
{
_diagnosticMessageSink.OnMessage(
new XunitDiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex}"));
throw;
}
}
}
return(runSummary);
}