Example #1
0
        protected override async Task <RunSummary> RunTestAsync()
        {
            var runSummary = new BenchmarkRunSummary
            {
                TestClassFullName       = TestCase.TestMethod.TestClass.Class.Name,
                TestClass               = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
                TestMethod              = TestCase.TestMethodName,
                Variation               = TestCase.Variation,
                ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
                RunStarted              = DateTime.UtcNow,
                MachineName             = _machineName,
                Framework               = _framework,
                Architecture            = IntPtr.Size > 4 ? "x64" : "x86",
                WarmupIterations        = TestCase.WarmupIterations,
                Iterations              = TestCase.Iterations,
                CustomData              = BenchmarkConfig.Instance.CustomData
            };

            for (var i = 0; i < TestCase.WarmupIterations; i++)
            {
                var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true);
                runSummary.Aggregate(await runner.RunAsync());
            }

            for (var i = 0; i < TestCase.Iterations; i++)
            {
                TestCase.MetricCollector.Reset();
                var runner           = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false);
                var iterationSummary = new BenchmarkIterationSummary();
                iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
                runSummary.Aggregate(iterationSummary);
            }

            if (runSummary.Failed != 0)
            {
                _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed."));
            }
            else
            {
                runSummary.PopulateMetrics();
                _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString()));

                foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
                {
                    try
                    {
                        new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
                    }
                    catch (Exception ex)
                    {
                        _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex}"));
                        throw;
                    }
                }
            }

            return(runSummary);
        }
        protected override async Task<RunSummary> RunTestAsync()
        {
            var runSummary = new BenchmarkRunSummary
            {
                TestClassFullName = TestCase.TestMethod.TestClass.Class.Name,
                TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
                TestMethod = TestCase.TestMethodName,
                Variation = TestCase.Variation,
                ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
                RunStarted = DateTime.UtcNow,
                MachineName = _machineName,
                Framework = _framework,
                Architecture = IntPtr.Size > 4 ? "x64" : "x86",
                WarmupIterations = TestCase.WarmupIterations,
                Iterations = TestCase.Iterations,
                CustomData = BenchmarkConfig.Instance.CustomData
            };

            for (int i = 0; i < TestCase.WarmupIterations; i++)
            {
                var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true);
                runSummary.Aggregate(await runner.RunAsync());
            }

            for (int i = 0; i < TestCase.Iterations; i++)
            {
                TestCase.MetricCollector.Reset();
                var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false);
                var iterationSummary = new BenchmarkIterationSummary();
                iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
                runSummary.Aggregate(iterationSummary);
            }

            if (runSummary.Failed != 0)
            {
                _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed."));
            }
            else
            {
                runSummary.PopulateMetrics();
                _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString()));

                foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
                {
                    try
                    {
                        new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
                    }
                    catch (Exception ex)
                    {
                        _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex.ToString()}"));
                        throw;
                    }
                }
            }

            return runSummary;
        }
        protected override async Task<RunSummary> RunTestAsync()
        {
            var runSummary = new BenchmarkRunSummary
            {
                TestClassFullName = TestCase.TestMethod.TestClass.Class.Name,
                TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
                TestMethod = TestCase.DisplayName,
                Variation = TestCase.Variation,
                ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
                RunStarted = DateTime.UtcNow,
                MachineName = _machineName,
                Framework = _framework,
                WarmupIterations = TestCase.WarmupIterations,
                Iterations = TestCase.Iterations,
                CustomData = BenchmarkConfig.Instance.CustomData
            };

            for (int i = 0; i < TestCase.WarmupIterations; i++)
            {
                var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true);
                runSummary.Aggregate(await runner.RunAsync());
            }

            for (int i = 0; i < TestCase.Iterations; i++)
            {
                TestCase.MetricCollector.Reset();
                var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false);
                var iterationSummary = new BenchmarkIterationSummary();
                iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
                runSummary.Aggregate(iterationSummary);
            }

            runSummary.PopulateMetrics();
            _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString()));

            foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
            {
                new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
            }

            return runSummary;
        }
Example #4
0
        protected override async Task <RunSummary> RunTestAsync()
        {
            var runSummary = new BenchmarkRunSummary
            {
                TestClassFullName       = TestCase.TestMethod.TestClass.Class.Name,
                TestClass               = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
                TestMethod              = TestCase.DisplayName,
                Variation               = TestCase.Variation,
                ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
                RunStarted              = DateTime.UtcNow,
                MachineName             = _machineName,
                Framework               = _framework,
                WarmupIterations        = TestCase.WarmupIterations,
                Iterations              = TestCase.Iterations,
                CustomData              = BenchmarkConfig.Instance.CustomData
            };

            for (int i = 0; i < TestCase.WarmupIterations; i++)
            {
                var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true);
                runSummary.Aggregate(await runner.RunAsync());
            }

            for (int i = 0; i < TestCase.Iterations; i++)
            {
                TestCase.MetricCollector.Reset();
                var runner           = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false);
                var iterationSummary = new BenchmarkIterationSummary();
                iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
                runSummary.Aggregate(iterationSummary);
            }

            runSummary.PopulateMetrics();
            _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString()));

            foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
            {
                new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
            }

            return(runSummary);
        }