[InlineData(2, 300)] // keep the values small since there's a real delay involved
        public void ShouldComputeMetricsCorrectly(int iterationCount, int millisecondRuntime)
        {
            var assertionOutput = new ActionBenchmarkOutput((report, warmup) =>
            {
                if (warmup) return;
                var counterResults = report.Metrics[CounterName];
                var projectedThroughput = 1000/(double)IterationSpeedMs; // roughly the max value of this counter
                var observedDifference =
                    Math.Abs(projectedThroughput - counterResults.MetricValuePerSecond);
                Assert.True(observedDifference <= 1.5d, $"delta between expected value and actual measured value should be <= 1.5, was {observedDifference} [{counterResults.MetricValuePerSecond} op /s]. Expected [{projectedThroughput} op /s]");
            }, results =>
            {
                var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max;
                Assert.Equal(iterationCount, counterResults);
            });

            var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty);
            var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
                Assertion.Empty);
            var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);

            var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Throughput, iterationCount, millisecondRuntime,
               new[] { gcBenchmark }, new[] { memoryBenchmark }, new[] { counterBenchmark });

            var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput);

            benchmark.Run();
        }
        public void ShouldComputeMetricsCorrectly(int iterationCount)
        {
            var assertionOutput = new ActionBenchmarkOutput((report, warmup) =>
            {
                if (!warmup)
                {
                    var counterResults = report.Metrics[CounterName];
                    Assert.Equal(1, counterResults.MetricValue);
                }
            }, results =>
            {
                var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max;
                Assert.Equal(iterationCount, counterResults);
            });

            var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty);
            var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
                Assertion.Empty);
            var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);

            var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, iterationCount, 1000,
               new[] { gcBenchmark }, new MemoryBenchmarkSetting[0], new[] {counterBenchmark});

            var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput);

            benchmark.Run();
        }
Пример #3
0
        public void ShouldComputeMetricsCorrectly(int iterationCount)
        {
            var assertionOutput = new ActionBenchmarkOutput((report, warmup) =>
            {
                if (!warmup)
                {
                    var counterResults = report.Metrics[CounterName];
                    Assert.Equal(1, counterResults.MetricValue);
                }
            }, results =>
            {
                var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max;
                Assert.Equal(iterationCount, counterResults);
            });

            var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty);
            var gcBenchmark      = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total,
                                                          Assertion.Empty);
            var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);

            var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, iterationCount, 1000,
                                                 new List <IBenchmarkSetting>()
            {
                gcBenchmark, counterBenchmark
            }, new Dictionary <MetricName, MetricsCollectorSelector>()
            {
                { gcBenchmark.MetricName, new GcCollectionsSelector() },
                { counterBenchmark.MetricName, new CounterSelector() }
            });

            var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput);

            benchmark.Run();
        }
Пример #4
0
        public void Should_build_when_at_least_one_metric_assigned()
        {
            var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
            var gcBenchmark      = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total,
                                                          Assertion.Empty);
            var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
            var settings        = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
                                                        new List <IBenchmarkSetting>()
            {
                gcBenchmark, memoryBenchmark, counterBenchmark
            },
                                                        new Dictionary <MetricName, MetricsCollectorSelector>()
            {
                { gcBenchmark.MetricName, new GcCollectionsSelector() },
                { counterBenchmark.MetricName, new CounterSelector() },
                { memoryBenchmark.MetricName, new TotalMemorySelector() }
            });

            var builder = new BenchmarkBuilder(settings);
            var run     = builder.NewRun(WarmupData.PreWarmup);

            Assert.Equal(3, run.MeasureCount);
            Assert.Equal(1, run.Counters.Count);
            Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
        }
Пример #5
0
        [InlineData(2, 300)]  // keep the values small since there's a real delay involved
        public void ShouldComputeMetricsCorrectly(int iterationCount, int millisecondRuntime)
        {
            var assertionOutput = new ActionBenchmarkOutput((report, warmup) =>
            {
                if (warmup)
                {
                    return;
                }
                var counterResults      = report.Metrics[CounterName];
                var projectedThroughput = 1000 / (double)IterationSpeedMs; // roughly the max value of this counter
                var observedDifference  =
                    Math.Abs(projectedThroughput - counterResults.MetricValuePerSecond);
                Assert.True(observedDifference <= 1.5d, $"delta between expected value and actual measured value should be <= 1.5, was {observedDifference} [{counterResults.MetricValuePerSecond} op /s]. Expected [{projectedThroughput} op /s]");
            }, results =>
            {
                var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max;
                Assert.Equal(iterationCount, counterResults);
            });

            var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty);
            var gcBenchmark      = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
                                                          Assertion.Empty);
            var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);

            var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Throughput, iterationCount, millisecondRuntime,
                                                 new[] { gcBenchmark }, new[] { memoryBenchmark }, new[] { counterBenchmark });

            var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput);

            benchmark.Run();
        }
 public void GcCollectionsSelector_should_create_1_GcCollectionsPerGenerationCollector_per_Generation()
 {
     var gcCollectionsSelector = new GcCollectionsSelector();
     var gcGenerations = SysInfo.Instance.MaxGcGeneration;
     var gcSetting = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
         Assertion.Empty);
     var gcCollectors = gcCollectionsSelector.Create(RunMode.Throughput, gcSetting).Cast<GcCollectionsPerGenerationCollector>().ToList();
     Assert.Equal(gcGenerations + 1,gcCollectors.Count);
     Assert.Equal(gcGenerations, gcCollectors.Max(x => x.Generation));
 }
Пример #7
0
        public void GcCollectionsSelector_should_create_1_GcCollectionsPerGenerationCollector_per_Generation()
        {
            var gcCollectionsSelector = new GcCollectionsSelector();
            var gcGenerations         = SysInfo.Instance.MaxGcGeneration;
            var gcSetting             = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
                                                               Assertion.Empty);
            var gcCollectors = gcCollectionsSelector.Create(RunMode.Throughput, gcSetting).Cast <GcCollectionsPerGenerationCollector>().ToList();

            Assert.Equal(gcGenerations + 1, gcCollectors.Count);
            Assert.Equal(gcGenerations, gcCollectors.Max(x => x.Generation));
        }
Пример #8
0
        public void Should_build_when_at_least_one_metric_assigned()
        {
            var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
            var gcBenchmark      = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
                                                          Assertion.Empty);
            var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
            var settings        = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
                                                        new[] { gcBenchmark }, new[] { memoryBenchmark }, new[] { counterBenchmark });

            var builder = new BenchmarkBuilder(settings);
            var run     = builder.NewRun(WarmupData.PreWarmup);

            Assert.Equal(2 + (SysInfo.Instance.MaxGcGeneration + 1), run.MeasureCount);
            Assert.Equal(1, run.Counters.Count);
            Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
        }
        public void Should_build_when_at_least_one_metric_assigned()
        {
            var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
            var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
                Assertion.Empty);
            var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
            var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
                new[] {gcBenchmark}, new[] { memoryBenchmark }, new[] { counterBenchmark });

            var builder = new BenchmarkBuilder(settings);
            var run = builder.NewRun(WarmupData.PreWarmup);

            Assert.Equal(2 + (SysInfo.Instance.MaxGcGeneration + 1), run.MeasureCount);
            Assert.Equal(1, run.Counters.Count);
            Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
        }
Пример #10
0
        public void Should_build_when_at_least_one_metric_assigned()
        {
            var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
            var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total,
                Assertion.Empty);
            var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
            var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
                new List<IBenchmarkSetting>(){gcBenchmark, memoryBenchmark, counterBenchmark },
                new Dictionary<MetricName, MetricsCollectorSelector>()
                {
                    { gcBenchmark.MetricName, new GcCollectionsSelector() },
                    { counterBenchmark.MetricName, new CounterSelector() },
                    { memoryBenchmark.MetricName, new TotalMemorySelector() }
                });

            var builder = new BenchmarkBuilder(settings);
            var run = builder.NewRun(WarmupData.PreWarmup);

            Assert.Equal(3, run.MeasureCount);
            Assert.Equal(1, run.Counters.Count);
            Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
        }