protected override async Task <RunSummary> RunTestAsync()
{
var runSummary = new BenchmarkRunSummary
{
TestClassFullName = TestCase.TestMethod.TestClass.Class.Name,
TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
TestMethod = TestCase.TestMethodName,
Variation = TestCase.Variation,
ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
RunStarted = DateTime.UtcNow,
MachineName = _machineName,
Framework = PlatformServices.Default.Runtime.RuntimeType,
Architecture = IntPtr.Size > 4 ? "x64" : "x86",
WarmupIterations = TestCase.WarmupIterations,
Iterations = TestCase.Iterations,
CustomData = BenchmarkConfig.Instance.CustomData
};
for (var i = 0; i < TestCase.WarmupIterations; i++)
{
var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true);
runSummary.Aggregate(await runner.RunAsync());
}
for (var i = 0; i < TestCase.Iterations; i++)
{
TestCase.MetricCollector.Reset();
var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false);
var iterationSummary = new BenchmarkIterationSummary();
iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
runSummary.Aggregate(iterationSummary);
}
if (runSummary.Failed != 0 || runSummary.Skipped != 0)
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed."));
}
else
{
runSummary.PopulateMetrics();
_diagnosticMessageSink.OnMessage(new DiagnosticMessage(runSummary.ToString()));
foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
{
try
{
new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
}
catch (Exception ex)
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex}"));
throw;
}
}
}
return(runSummary);
}
protected override async Task<RunSummary> RunTestAsync()
{
var runSummary = new BenchmarkRunSummary
{
TestClassFullName = TestCase.TestMethod.TestClass.Class.Name,
TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
TestMethod = TestCase.TestMethodName,
Variation = TestCase.Variation,
ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
RunStarted = DateTime.UtcNow,
MachineName = _machineName,
Framework = Microsoft.Extensions.Internal.RuntimeEnvironment.RuntimeType,
Architecture = IntPtr.Size > 4 ? "x64" : "x86",
WarmupIterations = TestCase.WarmupIterations,
Iterations = TestCase.Iterations,
CustomData = BenchmarkConfig.Instance.CustomData
};
for (var i = 0; i < TestCase.WarmupIterations; i++)
{
var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true);
runSummary.Aggregate(await runner.RunAsync());
}
for (var i = 0; i < TestCase.Iterations; i++)
{
TestCase.MetricCollector.Reset();
var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false);
var iterationSummary = new BenchmarkIterationSummary();
iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
runSummary.Aggregate(iterationSummary);
}
if (runSummary.Failed != 0 || runSummary.Skipped != 0)
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed."));
}
else
{
runSummary.PopulateMetrics();
_diagnosticMessageSink.OnMessage(new DiagnosticMessage(runSummary.ToString()));
foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
{
try
{
new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
}
catch (Exception ex)
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex}"));
throw;
}
}
}
return runSummary;
}
public void Aggregate(BenchmarkIterationSummary summary)
{
base.Aggregate(summary);
_iterations.Add(summary);
}