public void ShouldSkipWarmupsWhenSpecified(int iterationCount)
{
var observedWarmupCount = -1; //we have a pre-warmup that always happens no matter what. Need to account for it.
var assertionOutput = new ActionBenchmarkOutput((report, warmup) =>
{
if (warmup)
{
observedWarmupCount++;
}
}, results =>
{
Assert.Equal(1, observedWarmupCount);
});
var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, iterationCount, 1000,
new List <IBenchmarkSetting>()
{
counterBenchmark
}, new Dictionary <MetricName, MetricsCollectorSelector>()
{
{ counterBenchmark.MetricName, new CounterSelector() }
})
{
SkipWarmups = true
};
var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput);
benchmark.Run();
}
[InlineData(2, 300)] // keep the values small since there's a real delay involved
public void ShouldComputeMetricsCorrectly(int iterationCount, int millisecondRuntime)
{
var assertionOutput = new ActionBenchmarkOutput((report, warmup) =>
{
if (warmup) return;
var counterResults = report.Metrics[CounterName];
var projectedThroughput = 1000/(double)IterationSpeedMs; // roughly the max value of this counter
var observedDifference =
Math.Abs(projectedThroughput - counterResults.MetricValuePerSecond);
Assert.True(observedDifference <= 1.5d, $"delta between expected value and actual measured value should be <= 1.5, was {observedDifference} [{counterResults.MetricValuePerSecond} op /s]. Expected [{projectedThroughput} op /s]");
}, results =>
{
var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max;
Assert.Equal(iterationCount, counterResults);
});
var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty);
var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
Assertion.Empty);
var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Throughput, iterationCount, millisecondRuntime,
new[] { gcBenchmark }, new[] { memoryBenchmark }, new[] { counterBenchmark });
var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput);
benchmark.Run();
}
public void ShouldComputeMetricsCorrectly(int iterationCount)
{
var assertionOutput = new ActionBenchmarkOutput((report, warmup) =>
{
if (!warmup)
{
var counterResults = report.Metrics[CounterName];
Assert.Equal(1, counterResults.MetricValue);
}
}, results =>
{
var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max;
Assert.Equal(iterationCount, counterResults);
});
var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty);
var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total,
Assertion.Empty);
var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, iterationCount, 1000,
new List<IBenchmarkSetting>(){ gcBenchmark, counterBenchmark }, new Dictionary<MetricName, MetricsCollectorSelector>() { { gcBenchmark.MetricName, new GcCollectionsSelector() },
{ counterBenchmark.MetricName, new CounterSelector() } });
var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput);
benchmark.Run();
}
public void ShouldComputeMetricsCorrectly(int iterationCount)
{
var assertionOutput = new ActionBenchmarkOutput((report, warmup) =>
{
if (!warmup)
{
var counterResults = report.Metrics[CounterName];
Assert.Equal(1, counterResults.MetricValue);
}
}, results =>
{
var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max;
Assert.Equal(iterationCount, counterResults);
});
var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty);
var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total,
Assertion.Empty);
var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, iterationCount, 1000,
new List <IBenchmarkSetting>()
{
gcBenchmark, counterBenchmark
}, new Dictionary <MetricName, MetricsCollectorSelector>()
{
{ gcBenchmark.MetricName, new GcCollectionsSelector() },
{ counterBenchmark.MetricName, new CounterSelector() }
});
var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput);
benchmark.Run();
}
public void Should_build_when_at_least_one_metric_assigned()
{
var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total,
Assertion.Empty);
var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
new List <IBenchmarkSetting>()
{
gcBenchmark, memoryBenchmark, counterBenchmark
},
new Dictionary <MetricName, MetricsCollectorSelector>()
{
{ gcBenchmark.MetricName, new GcCollectionsSelector() },
{ counterBenchmark.MetricName, new CounterSelector() },
{ memoryBenchmark.MetricName, new TotalMemorySelector() }
});
var builder = new BenchmarkBuilder(settings);
var run = builder.NewRun(WarmupData.PreWarmup);
Assert.Equal(3, run.MeasureCount);
Assert.Equal(1, run.Counters.Count);
Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
}
[InlineData(2, 300)] // keep the values small since there's a real delay involved
public void ShouldComputeMetricsCorrectly(int iterationCount, int millisecondRuntime)
{
var assertionOutput = new ActionBenchmarkOutput((report, warmup) =>
{
if (warmup)
{
return;
}
var counterResults = report.Metrics[CounterName];
var projectedThroughput = 1000 / (double)IterationSpeedMs; // roughly the max value of this counter
var observedDifference =
Math.Abs(projectedThroughput - counterResults.MetricValuePerSecond);
Assert.True(observedDifference <= 1.5d, $"delta between expected value and actual measured value should be <= 1.5, was {observedDifference} [{counterResults.MetricValuePerSecond} op /s]. Expected [{projectedThroughput} op /s]");
}, results =>
{
var counterResults = results.Data.StatsByMetric[CounterName].Stats.Max;
Assert.Equal(iterationCount, counterResults);
});
var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty);
var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
Assertion.Empty);
var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Throughput, iterationCount, millisecondRuntime,
new[] { gcBenchmark }, new[] { memoryBenchmark }, new[] { counterBenchmark });
var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput);
benchmark.Run();
}
public void Should_build_when_exactly_one_metric_assigned()
{
var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
new GcBenchmarkSetting[0], new MemoryBenchmarkSetting[0], new CounterBenchmarkSetting[] { counterBenchmark });
var builder = new BenchmarkBuilder(settings);
var run = builder.NewRun(WarmupData.PreWarmup);
Assert.Equal(1, run.MeasureCount);
Assert.Equal(1, run.Counters.Count);
Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
}
public void Should_build_when_exactly_one_metric_assigned()
{
var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
new GcBenchmarkSetting[0], new MemoryBenchmarkSetting[0], new CounterBenchmarkSetting[] { counterBenchmark});
var builder = new BenchmarkBuilder(settings);
var run = builder.NewRun(WarmupData.PreWarmup);
Assert.Equal(1, run.MeasureCount);
Assert.Equal(1, run.Counters.Count);
Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
}
public void Should_build_when_at_least_one_metric_assigned()
{
var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
Assertion.Empty);
var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
new[] { gcBenchmark }, new[] { memoryBenchmark }, new[] { counterBenchmark });
var builder = new BenchmarkBuilder(settings);
var run = builder.NewRun(WarmupData.PreWarmup);
Assert.Equal(2 + (SysInfo.Instance.MaxGcGeneration + 1), run.MeasureCount);
Assert.Equal(1, run.Counters.Count);
Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
}
public void Should_build_when_at_least_one_metric_assigned()
{
var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
Assertion.Empty);
var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
new[] {gcBenchmark}, new[] { memoryBenchmark }, new[] { counterBenchmark });
var builder = new BenchmarkBuilder(settings);
var run = builder.NewRun(WarmupData.PreWarmup);
Assert.Equal(2 + (SysInfo.Instance.MaxGcGeneration + 1), run.MeasureCount);
Assert.Equal(1, run.Counters.Count);
Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
}
public void Should_build_when_at_least_one_metric_assigned()
{
var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total,
Assertion.Empty);
var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
new List<IBenchmarkSetting>(){gcBenchmark, memoryBenchmark, counterBenchmark },
new Dictionary<MetricName, MetricsCollectorSelector>()
{
{ gcBenchmark.MetricName, new GcCollectionsSelector() },
{ counterBenchmark.MetricName, new CounterSelector() },
{ memoryBenchmark.MetricName, new TotalMemorySelector() }
});
var builder = new BenchmarkBuilder(settings);
var run = builder.NewRun(WarmupData.PreWarmup);
Assert.Equal(3, run.MeasureCount);
Assert.Equal(1, run.Counters.Count);
Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
}
public void ShouldExecuteCorrectWarmupCount(int iterationCount)
{
var observedWarmupCount = -1; //we have a pre-warmup that always happens no matter what. Need to account for it.
var assertionOutput = new ActionBenchmarkOutput((report, warmup) =>
{
if (warmup)
{
observedWarmupCount++;
}
}, results =>
{
Assert.Equal(iterationCount, observedWarmupCount);
});
var counterBenchmark = new CounterBenchmarkSetting(CounterName.CounterName, AssertionType.Total, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, iterationCount, 1000,
new List<IBenchmarkSetting>() { counterBenchmark }, new Dictionary<MetricName, MetricsCollectorSelector>() {
{ counterBenchmark.MetricName, new CounterSelector() } });
var benchmark = new Benchmark(settings, _benchmarkMethods, assertionOutput);
benchmark.Run();
}
public CreateCounterBenchmarkSetting(CounterBenchmarkSetting benchmarkSetting, AtomicCounter counter)
{
BenchmarkSetting = benchmarkSetting;
Counter = counter;
}