public void ShouldSkipWarmupsWhenSpecified(int iterationCount) { var observedWarmupCount = -1; //we have a pre-warmup that always happens no matter what. Need to account for it. var assertionOutput = new ActionBenchmarkOutput((report, warmup) => { if (warmup) { observedWarmupCount++; } }, results => { Assert.Equal(1, observedWarmupCount); }); var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, iterationCount, 1000, new List <IBenchmarkSetting>() { counterBenchmark }, new Dictionary <MetricName, MetricsCollectorSelector>() { { counterBenchmark.MetricName, new CounterSelector() } }) { SkipWarmups = true }; var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput); benchmark.Run(); }
[InlineData(2, 300)] // keep the values small since there's a real delay involved public void ShouldComputeMetricsCorrectly(int iterationCount, int millisecondRuntime) { var assertionOutput = new ActionBenchmarkOutput((report, warmup) => { if (warmup) return; var counterResults = report.Metrics[CounterName]; var projectedThroughput = 1000/(double)IterationSpeedMs; // roughly the max value of this counter var observedDifference = Math.Abs(projectedThroughput - counterResults.MetricValuePerSecond); Assert.True(observedDifference <= 1.5d, $"delta between expected value and actual measured value should be <= 1.5, was {observedDifference} [{counterResults.MetricValuePerSecond} op /s]. Expected [{projectedThroughput} op /s]"); }, results => { var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max; Assert.Equal(iterationCount, counterResults); }); var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty); var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total, Assertion.Empty); var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Throughput, iterationCount, millisecondRuntime, new[] { gcBenchmark }, new[] { memoryBenchmark }, new[] { counterBenchmark }); var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput); benchmark.Run(); }
public void ShouldComputeMetricsCorrectly(int iterationCount) { var assertionOutput = new ActionBenchmarkOutput((report, warmup) => { if (!warmup) { var counterResults = report.Metrics[CounterName]; Assert.Equal(1, counterResults.MetricValue); } }, results => { var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max; Assert.Equal(iterationCount, counterResults); }); var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty); var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total, Assertion.Empty); var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, iterationCount, 1000, new List<IBenchmarkSetting>(){ gcBenchmark, counterBenchmark }, new Dictionary<MetricName, MetricsCollectorSelector>() { { gcBenchmark.MetricName, new GcCollectionsSelector() }, { counterBenchmark.MetricName, new CounterSelector() } }); var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput); benchmark.Run(); }
public void ShouldComputeMetricsCorrectly(int iterationCount) { var assertionOutput = new ActionBenchmarkOutput((report, warmup) => { if (!warmup) { var counterResults = report.Metrics[CounterName]; Assert.Equal(1, counterResults.MetricValue); } }, results => { var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max; Assert.Equal(iterationCount, counterResults); }); var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty); var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total, Assertion.Empty); var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, iterationCount, 1000, new List <IBenchmarkSetting>() { gcBenchmark, counterBenchmark }, new Dictionary <MetricName, MetricsCollectorSelector>() { { gcBenchmark.MetricName, new GcCollectionsSelector() }, { counterBenchmark.MetricName, new CounterSelector() } }); var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput); benchmark.Run(); }
public void Should_build_when_at_least_one_metric_assigned() { var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty); var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total, Assertion.Empty); var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000, new List <IBenchmarkSetting>() { gcBenchmark, memoryBenchmark, counterBenchmark }, new Dictionary <MetricName, MetricsCollectorSelector>() { { gcBenchmark.MetricName, new GcCollectionsSelector() }, { counterBenchmark.MetricName, new CounterSelector() }, { memoryBenchmark.MetricName, new TotalMemorySelector() } }); var builder = new BenchmarkBuilder(settings); var run = builder.NewRun(WarmupData.PreWarmup); Assert.Equal(3, run.MeasureCount); Assert.Equal(1, run.Counters.Count); Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName)); }
[InlineData(2, 300)] // keep the values small since there's a real delay involved public void ShouldComputeMetricsCorrectly(int iterationCount, int millisecondRuntime) { var assertionOutput = new ActionBenchmarkOutput((report, warmup) => { if (warmup) { return; } var counterResults = report.Metrics[CounterName]; var projectedThroughput = 1000 / (double)IterationSpeedMs; // roughly the max value of this counter var observedDifference = Math.Abs(projectedThroughput - counterResults.MetricValuePerSecond); Assert.True(observedDifference <= 1.5d, $"delta between expected value and actual measured value should be <= 1.5, was {observedDifference} [{counterResults.MetricValuePerSecond} op /s]. Expected [{projectedThroughput} op /s]"); }, results => { var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max; Assert.Equal(iterationCount, counterResults); }); var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty); var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total, Assertion.Empty); var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Throughput, iterationCount, millisecondRuntime, new[] { gcBenchmark }, new[] { memoryBenchmark }, new[] { counterBenchmark }); var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput); benchmark.Run(); }
public void Should_build_when_exactly_one_metric_assigned() { var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000, new GcBenchmarkSetting[0], new MemoryBenchmarkSetting[0], new CounterBenchmarkSetting[] { counterBenchmark }); var builder = new BenchmarkBuilder(settings); var run = builder.NewRun(WarmupData.PreWarmup); Assert.Equal(1, run.MeasureCount); Assert.Equal(1, run.Counters.Count); Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName)); }
public void Should_build_when_exactly_one_metric_assigned() { var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000, new GcBenchmarkSetting[0], new MemoryBenchmarkSetting[0], new CounterBenchmarkSetting[] { counterBenchmark}); var builder = new BenchmarkBuilder(settings); var run = builder.NewRun(WarmupData.PreWarmup); Assert.Equal(1, run.MeasureCount); Assert.Equal(1, run.Counters.Count); Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName)); }
public void Should_build_when_at_least_one_metric_assigned() { var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty); var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total, Assertion.Empty); var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000, new[] { gcBenchmark }, new[] { memoryBenchmark }, new[] { counterBenchmark }); var builder = new BenchmarkBuilder(settings); var run = builder.NewRun(WarmupData.PreWarmup); Assert.Equal(2 + (SysInfo.Instance.MaxGcGeneration + 1), run.MeasureCount); Assert.Equal(1, run.Counters.Count); Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName)); }
public void Should_build_when_at_least_one_metric_assigned() { var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty); var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total, Assertion.Empty); var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000, new[] {gcBenchmark}, new[] { memoryBenchmark }, new[] { counterBenchmark }); var builder = new BenchmarkBuilder(settings); var run = builder.NewRun(WarmupData.PreWarmup); Assert.Equal(2 + (SysInfo.Instance.MaxGcGeneration + 1), run.MeasureCount); Assert.Equal(1, run.Counters.Count); Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName)); }
public void Should_build_when_at_least_one_metric_assigned() { var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty); var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total, Assertion.Empty); var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000, new List<IBenchmarkSetting>(){gcBenchmark, memoryBenchmark, counterBenchmark }, new Dictionary<MetricName, MetricsCollectorSelector>() { { gcBenchmark.MetricName, new GcCollectionsSelector() }, { counterBenchmark.MetricName, new CounterSelector() }, { memoryBenchmark.MetricName, new TotalMemorySelector() } }); var builder = new BenchmarkBuilder(settings); var run = builder.NewRun(WarmupData.PreWarmup); Assert.Equal(3, run.MeasureCount); Assert.Equal(1, run.Counters.Count); Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName)); }
public void ShouldExecuteCorrectWarmupCount(int iterationCount) { var observedWarmupCount = -1; //we have a pre-warmup that always happens no matter what. Need to account for it. var assertionOutput = new ActionBenchmarkOutput((report, warmup) => { if (warmup) { observedWarmupCount++; } }, results => { Assert.Equal(iterationCount, observedWarmupCount); }); var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty); var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, iterationCount, 1000, new List<IBenchmarkSetting>() { counterBenchmark }, new Dictionary<MetricName, MetricsCollectorSelector>() { { counterBenchmark.MetricName, new CounterSelector() } }); var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput); benchmark.Run(); }
public CreateCounterBenchmarkSetting(CounterBenchmarkSetting benchmarkSetting, AtomicCounter counter) { BenchmarkSetting = benchmarkSetting; Counter = counter; }