protected override async Task <RunSummary> RunTestAsync() { var runSummary = new BenchmarkRunSummary { TestClassFullName = TestCase.TestMethod.TestClass.Class.Name, TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(), TestMethod = TestCase.TestMethodName, Variation = TestCase.Variation, ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion, RunStarted = DateTime.UtcNow, MachineName = _machineName, Framework = _framework, Architecture = IntPtr.Size > 4 ? "x64" : "x86", WarmupIterations = TestCase.WarmupIterations, Iterations = TestCase.Iterations, CustomData = BenchmarkConfig.Instance.CustomData }; for (var i = 0; i < TestCase.WarmupIterations; i++) { var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true); runSummary.Aggregate(await runner.RunAsync()); } for (var i = 0; i < TestCase.Iterations; i++) { TestCase.MetricCollector.Reset(); var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false); var iterationSummary = new BenchmarkIterationSummary(); iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector); runSummary.Aggregate(iterationSummary); } if (runSummary.Failed != 0) { _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed.")); } else { runSummary.PopulateMetrics(); _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString())); foreach (var database in BenchmarkConfig.Instance.ResultDatabases) { try { new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary); } catch (Exception ex) { _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex}")); throw; } } } return(runSummary); }
protected override async Task<RunSummary> RunTestAsync() { var runSummary = new BenchmarkRunSummary { TestClassFullName = TestCase.TestMethod.TestClass.Class.Name, TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(), TestMethod = TestCase.TestMethodName, Variation = TestCase.Variation, ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion, RunStarted = DateTime.UtcNow, MachineName = _machineName, Framework = _framework, Architecture = IntPtr.Size > 4 ? "x64" : "x86", WarmupIterations = TestCase.WarmupIterations, Iterations = TestCase.Iterations, CustomData = BenchmarkConfig.Instance.CustomData }; for (int i = 0; i < TestCase.WarmupIterations; i++) { var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true); runSummary.Aggregate(await runner.RunAsync()); } for (int i = 0; i < TestCase.Iterations; i++) { TestCase.MetricCollector.Reset(); var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false); var iterationSummary = new BenchmarkIterationSummary(); iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector); runSummary.Aggregate(iterationSummary); } if (runSummary.Failed != 0) { _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed.")); } else { runSummary.PopulateMetrics(); _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString())); foreach (var database in BenchmarkConfig.Instance.ResultDatabases) { try { new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary); } catch (Exception ex) { _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex.ToString()}")); throw; } } } return runSummary; }
protected override async Task<RunSummary> RunTestAsync() { var runSummary = new BenchmarkRunSummary { TestClassFullName = TestCase.TestMethod.TestClass.Class.Name, TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(), TestMethod = TestCase.DisplayName, Variation = TestCase.Variation, ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion, RunStarted = DateTime.UtcNow, MachineName = _machineName, Framework = _framework, WarmupIterations = TestCase.WarmupIterations, Iterations = TestCase.Iterations, CustomData = BenchmarkConfig.Instance.CustomData }; for (int i = 0; i < TestCase.WarmupIterations; i++) { var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true); runSummary.Aggregate(await runner.RunAsync()); } for (int i = 0; i < TestCase.Iterations; i++) { TestCase.MetricCollector.Reset(); var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false); var iterationSummary = new BenchmarkIterationSummary(); iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector); runSummary.Aggregate(iterationSummary); } runSummary.PopulateMetrics(); _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString())); foreach (var database in BenchmarkConfig.Instance.ResultDatabases) { new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary); } return runSummary; }
protected override async Task <RunSummary> RunTestAsync() { var runSummary = new BenchmarkRunSummary { TestClassFullName = TestCase.TestMethod.TestClass.Class.Name, TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(), TestMethod = TestCase.DisplayName, Variation = TestCase.Variation, ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion, RunStarted = DateTime.UtcNow, MachineName = _machineName, Framework = _framework, WarmupIterations = TestCase.WarmupIterations, Iterations = TestCase.Iterations, CustomData = BenchmarkConfig.Instance.CustomData }; for (int i = 0; i < TestCase.WarmupIterations; i++) { var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true); runSummary.Aggregate(await runner.RunAsync()); } for (int i = 0; i < TestCase.Iterations; i++) { TestCase.MetricCollector.Reset(); var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false); var iterationSummary = new BenchmarkIterationSummary(); iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector); runSummary.Aggregate(iterationSummary); } runSummary.PopulateMetrics(); _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString())); foreach (var database in BenchmarkConfig.Instance.ResultDatabases) { new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary); } return(runSummary); }
public void Aggregate(BenchmarkIterationSummary summary) { base.Aggregate(summary); _iterations.Add(summary); }