/// <summary>
/// Generates a new <see cref="BenchmarkRun" /> based on the provided settings, available system metrics,
/// and (optionally) the duration of the last run.
/// </summary>
/// <param name="warmupData">Data collected during warm-up</param>
/// <returns>A new <see cref="BenchmarkRun" /> instance.</returns>
public BenchmarkRun NewRun(WarmupData warmupData)
{
var numberOfMetrics = Settings.TotalTrackedMetrics;
var measurements = new List <MeasureBucket>(numberOfMetrics);
var counterSettings = Settings.CounterMeasurements.ToList();
var counters = new List <Counter>(counterSettings.Count);
// need to exclude counters first
var settingsExceptCounters = Settings.DistinctMeasurements.Except(counterSettings);
foreach (var setting in settingsExceptCounters)
{
var selector = Settings.Collectors[setting.MetricName];
var collector = selector.Create(Settings.RunMode, warmupData, setting);
measurements.Add(new MeasureBucket(collector));
}
foreach (var counterSetting in counterSettings)
{
var setting = counterSetting;
var selector = Settings.Collectors[setting.MetricName];
var atomicCounter = new AtomicCounter();
var createCounterBenchmark = new CreateCounterBenchmarkSetting(setting, atomicCounter);
var collector = selector.Create(Settings.RunMode, warmupData, createCounterBenchmark);
measurements.Add(new MeasureBucket(collector));
counters.Add(new Counter(atomicCounter, setting.CounterName));
}
return(new BenchmarkRun(measurements, counters, Settings.Trace));
}
public override MetricCollector Create(RunMode runMode, WarmupData warmup, IBenchmarkSetting setting)
{
var timingSetting = setting as TimingBenchmarkSetting;
Contract.Assert(timingSetting != null);
return new TimingCollector(timingSetting.TimingMetricName);
}
public override IEnumerable<MetricCollector> Create(RunMode runMode, WarmupData warmup, IBenchmarkSetting setting)
{
//if (warmup.ElapsedTicks <= BenchmarkConstants.SamplingPrecisionTicks)
return new[] {new GcTotalMemoryCollector(MemoryMetricName)};
//return new[] {new PerformanceCounterTotalMemoryCollector(MemoryMetricName)};
}
public override MetricCollector Create(RunMode runMode, WarmupData warmup, IBenchmarkSetting setting)
{
Contract.Assert(setting != null);
Contract.Assert(setting is CreateCounterBenchmarkSetting);
var createCounter = setting as CreateCounterBenchmarkSetting;
// ReSharper disable once PossibleNullReferenceException
// resolved with Code Contracts
return new CounterMetricCollector(createCounter.BenchmarkSetting.CounterName, createCounter.Counter);
}
public override MetricCollector Create(RunMode runMode, WarmupData warmup,
IBenchmarkSetting setting)
{
Contract.Assert(setting != null);
Contract.Assert(setting is GcBenchmarkSetting);
var gcSetting = setting as GcBenchmarkSetting;
// ReSharper disable once PossibleNullReferenceException
// covered by Code Contracts
if (gcSetting.Generation == GcGeneration.AllGc)
{
throw new InvalidOperationException($"{gcSetting.Generation} is not supported by this collector");
}
return CreateInstanceInternal((int) gcSetting.Generation);
}
/// <summary>
/// Generates a new <see cref="BenchmarkRun" /> based on the provided settings, available system metrics,
/// and (optionally) the duration of the last run.
/// </summary>
/// <param name="warmupData">Data collected during warm-up</param>
/// <returns>A new <see cref="BenchmarkRun" /> instance.</returns>
public BenchmarkRun NewRun(WarmupData warmupData)
{
var numberOfMetrics = Settings.TotalTrackedMetrics;
var measurements = new List <MeasureBucket>(numberOfMetrics);
var counters = new List <Counter>(Settings.CounterBenchmarks.Count);
for (var i = 0; i < Settings.DistinctMemoryBenchmarks.Count; i++)
{
var setting = Settings.DistinctMemoryBenchmarks[i];
var collectors = MemorySelectors[setting.Metric].Create(Settings.RunMode, warmupData, setting);
foreach (var collector in collectors)
{
measurements.Add(new MeasureBucket(collector));
}
}
for (var i = 0; i < Settings.DistinctGcBenchmarks.Count; i++)
{
var setting = Settings.DistinctGcBenchmarks[i];
var collectors = GcSelectors[setting.Metric].Create(Settings.RunMode, warmupData, setting);
foreach (var collector in collectors)
{
measurements.Add(new MeasureBucket(collector));
}
}
for (var i = 0; i < Settings.DistinctCounterBenchmarks.Count; i++)
{
var setting = Settings.DistinctCounterBenchmarks[i];
var atomicCounter = new AtomicCounter();
var createCounterBenchmark = new CreateCounterBenchmarkSetting(setting, atomicCounter);
var collectors = CounterSelector.Create(Settings.RunMode, warmupData, createCounterBenchmark);
foreach (var collector in collectors)
{
measurements.Add(new MeasureBucket(collector));
counters.Add(new Counter(atomicCounter, setting.CounterName));
}
}
return(new BenchmarkRun(measurements, counters));
}
public override IEnumerable<MetricCollector> Create(RunMode runMode, WarmupData warmup,
IBenchmarkSetting setting)
{
Contract.Assert(setting != null);
Contract.Assert(setting is GcBenchmarkSetting);
var gcSetting = setting as GcBenchmarkSetting;
// ReSharper disable once PossibleNullReferenceException
// covered by Code Contracts
if (gcSetting.Generation == GcGeneration.AllGc)
{
var collectors = new List<MetricCollector>(SystemInfo.MaxGcGeneration + 1);
for (var i = 0; i <= SystemInfo.MaxGcGeneration; i++)
{
collectors.Add(CreateInstanceInternal(i));
}
return collectors;
}
return new[] {CreateInstanceInternal((int) gcSetting.Generation)};
}
/// <summary>
/// Generates a new <see cref="BenchmarkRun" /> based on the provided settings, available system metrics,
/// and (optionally) the duration of the last run.
/// </summary>
/// <param name="warmupData">Data collected during warm-up</param>
/// <returns>A new <see cref="BenchmarkRun" /> instance.</returns>
public BenchmarkRun NewRun(WarmupData warmupData)
{
var numberOfMetrics = Settings.TotalTrackedMetrics;
var measurements = new List<MeasureBucket>(numberOfMetrics);
var counters = new List<Counter>(Settings.CounterBenchmarks.Count);
for (var i = 0; i < Settings.DistinctMemoryBenchmarks.Count; i++)
{
var setting = Settings.DistinctMemoryBenchmarks[i];
var collectors = MemorySelectors[setting.Metric].Create(Settings.RunMode, warmupData, setting);
foreach (var collector in collectors)
measurements.Add(new MeasureBucket(collector));
}
for (var i = 0; i < Settings.DistinctGcBenchmarks.Count; i++)
{
var setting = Settings.DistinctGcBenchmarks[i];
var collectors = GcSelectors[setting.Metric].Create(Settings.RunMode, warmupData, setting);
foreach (var collector in collectors)
measurements.Add(new MeasureBucket(collector));
}
for (var i = 0; i < Settings.DistinctCounterBenchmarks.Count; i++)
{
var setting = Settings.DistinctCounterBenchmarks[i];
var atomicCounter = new AtomicCounter();
var createCounterBenchmark = new CreateCounterBenchmarkSetting(setting, atomicCounter);
var collectors = CounterSelector.Create(Settings.RunMode, warmupData, createCounterBenchmark);
foreach (var collector in collectors)
{
measurements.Add(new MeasureBucket(collector));
counters.Add(new Counter(atomicCounter, setting.CounterName));
}
}
return new BenchmarkRun(measurements, counters);
}
public bool Equals(WarmupData other)
{
return ElapsedTicks == other.ElapsedTicks && ActualRunsMeasured == other.ActualRunsMeasured;
}
/// <summary>
/// Warmup phase
/// </summary>
private void WarmUp()
{
var warmupStopWatch = new Stopwatch();
var targetTime = Settings.RunTime;
Contract.Assert(targetTime != TimeSpan.Zero);
var runCount = 0L;
/* Pre-Warmup */
RunSingleBenchmark();
// check to see if pre-warmup threw an exception
var faulted = _currentRun.IsFaulted;
if (faulted)
{
/*
* Normally we don't ever queue up the warmup into the final stats, but we do it
* in failure cases so we can capture the exception thrown during warmup into
* the final report we're going to deliver to the end-user.
*/
CompletedRuns.Enqueue(_currentRun.ToReport(TimeSpan.Zero));
return;
}
/* Esimate */
Allocate(); // allocate all collectors needed
PreRun();
if (Settings.RunMode == RunMode.Throughput)
{
warmupStopWatch.Start();
while (warmupStopWatch.ElapsedTicks < targetTime.Ticks)
{
Invoker.InvokeRun(_currentRun.Context);
runCount++;
}
warmupStopWatch.Stop();
}
else
{
warmupStopWatch.Start();
Invoker.InvokeRun(_currentRun.Context);
runCount++;
warmupStopWatch.Stop();
}
PostRun();
Complete();
// elapsed time
var runTime = warmupStopWatch.ElapsedTicks;
WarmupData = new WarmupData(runTime, runCount);
var i = WarmupCount;
/* Warmup to force CPU caching */
while (i > 0 && !_currentRun.IsFaulted)
{
RunSingleBenchmark();
i--;
}
}
public override MetricCollector Create(RunMode runMode, WarmupData warmup, IBenchmarkSetting setting)
{
Contract.Assert(setting != null);
Contract.Assert(setting is PerformanceCounterBenchmarkSetting);
var counterBenchmarkSetting = setting as PerformanceCounterBenchmarkSetting;
var name = counterBenchmarkSetting.PerformanceCounterMetric;
// re-use the PerformanceCounter objects in our pool if possible
if (_cache.Exists(name))
return new PerformanceCounterValueCollector(name, name.UnitName ?? MetricNames.DefaultUnitName, _cache.Get(name), true);
// otherwise, warm up new ones
var maxRetries = 3;
var currentRetries = 0;
if (!PerformanceCounterCategory.CounterExists(name.CounterName, name.CategoryName))
throw new NBenchException($"Performance counter {name.ToHumanFriendlyString()} is not registered on this machine. Please create it first.");
// check to see that the instance we're interested in is registered
if (!string.IsNullOrEmpty(name.InstanceName))
{
var categories = PerformanceCounterCategory.GetCategories().Where(x => x.CategoryType == PerformanceCounterCategoryType.MultiInstance).ToList();
#if DEBUG
Console.WriteLine("---- DEBUG -----");
Console.WriteLine("{0} multi-instance categories detected", categories.Count);
#endif
var category = categories.Single(x => x.CategoryName == name.CategoryName);
var instances = category.GetInstanceNames();
if (!instances.Contains(name.InstanceName))
{
#if DEBUG
Console.WriteLine("---- DEBUG -----");
Console.WriteLine("Multi-instance? {0}", category.CategoryType);
foreach (var instance in instances)
Console.WriteLine(instance);
#endif
throw new NBenchException($"Performance counter {name.CategoryName}:{name.CounterName} exists, but we could not find an instance {name.InstanceName}.");
}
}
var proxy = new PerformanceCounterProxy(() =>
{
var counter = new PerformanceCounter(name.CategoryName, name.CounterName,
name.InstanceName ?? string.Empty, true);
return counter;
});
while (!CanFindPerformanceCounter(name) && currentRetries <= maxRetries)
{
Thread.Sleep(TimeSpan.FromMilliseconds(1000 + 100 * (currentRetries ^ 2))); // little bit of exponential backoff
if (proxy.CanWarmup)
break;
currentRetries++;
}
if (!proxy.CanWarmup)
throw new NBenchException($"Performance counter {name.ToHumanFriendlyString()} is not registered on this machine. Please create it first.");
// cache this performance counter and pool it for re-use
_cache.Put(name, proxy);
return new PerformanceCounterValueCollector(name, name.UnitName ?? MetricNames.DefaultUnitName, _cache.Get(name), true);
}
/// <summary>
/// Warmup phase
/// </summary>
private void WarmUp()
{
var warmupStopWatch = new Stopwatch();
var targetTime = Settings.RunTime;
Contract.Assert(targetTime != TimeSpan.Zero);
var runCount = 0L;
/* Pre-Warmup */
RunSingleBenchmark();
// check to see if pre-warmup threw an exception
var faulted = _currentRun.IsFaulted;
if (faulted)
{
/*
* Normally we don't ever queue up the warmup into the final stats, but we do it
* in failure cases so we can capture the exception thrown during warmup into
* the final report we're going to deliver to the end-user.
*/
CompletedRuns.Enqueue(_currentRun.ToReport(TimeSpan.Zero));
return;
}
/* Esimate */
Allocate(); // allocate all collectors needed
PreRun();
if (Settings.RunMode == RunMode.Throughput)
{
warmupStopWatch.Start();
while (warmupStopWatch.ElapsedTicks < targetTime.Ticks)
{
Invoker.InvokeRun(_currentRun.Context);
runCount++;
}
warmupStopWatch.Stop();
}
else
{
warmupStopWatch.Start();
Invoker.InvokeRun(_currentRun.Context);
runCount++;
warmupStopWatch.Stop();
}
PostRun();
Complete();
// elapsed time
var runTime = warmupStopWatch.ElapsedTicks;
WarmupData = new WarmupData(runTime, runCount);
var i = WarmupCount;
/* Warmup to force CPU caching */
while (i > 0 && !_currentRun.IsFaulted)
{
RunSingleBenchmark();
i--;
}
}
/// <summary>
/// Warmup phase
/// </summary>
private void WarmUp()
{
Trace.Debug($"Beginning Warmups for {BenchmarkName}");
var warmupStopWatch = new Stopwatch();
var targetTime = Settings.RunTime;
Contract.Assert(targetTime != TimeSpan.Zero);
var runCount = 0L;
var runTime = 0L;
/* Pre-Warmup */
Trace.Debug("----- BEGIN PRE-WARMUP -----");
/* Estimate */
Allocate(); // allocate all collectors needed
PreRun();
try
{
if (Settings.RunMode == RunMode.Throughput)
{
Trace.Debug(
$"Throughput mode: estimating how many invocations of {BenchmarkName} will take {targetTime.TotalSeconds}s");
var estimateCount = 3;
var runEstimates = new long[estimateCount];
var timeEstimates = new long[estimateCount];
for (var i = 0; i <= estimateCount; i++)
{
warmupStopWatch.Start();
while (warmupStopWatch.ElapsedTicks < targetTime.Ticks)
{
Invoker.InvokeRun(_currentRun.Context);
runCount++;
}
warmupStopWatch.Stop();
if (i > 0)
{
runEstimates[i - 1] = runCount;
timeEstimates[i - 1] = warmupStopWatch.ElapsedTicks;
}
runCount = 0;
warmupStopWatch.Reset();
}
runCount = (long)Math.Ceiling(runEstimates.Average());
runTime = (long) Math.Ceiling(timeEstimates.Average());
Trace.Debug(
$"Throughput mode: executed {runCount} instances of {BenchmarkName} in roughly {targetTime.TotalSeconds}s. Using that figure for benchmark.");
}
else
{
warmupStopWatch.Start();
Invoker.InvokeRun(_currentRun.Context);
runCount++;
warmupStopWatch.Stop();
// elapsed time
runTime = warmupStopWatch.ElapsedTicks;
}
}
catch (Exception ex)
{
HandleBenchmarkRunException(ex, $"Error occurred during ${BenchmarkName} RUN.");
}
PostRun();
Complete(true);
// check to see if pre-warmup threw an exception
var faulted = _currentRun.IsFaulted;
if (faulted)
{
Trace.Error($"Error occurred during pre-warmup. Exiting and producing dump...");
/*
* Normally we don't ever queue up the warmup into the final stats, but we do it
* in failure cases so we can capture the exception thrown during warmup into
* the final report we're going to deliver to the end-user.
*/
CompletedRuns.Enqueue(_currentRun.ToReport(TimeSpan.Zero));
return;
}
Trace.Debug("----- END PRE-WARMUP -----");
WarmupData = new WarmupData(runTime, runCount);
if (!Settings.SkipWarmups)
{
Trace.Debug("----- BEGIN WARMUPS -----");
var i = _warmupCount;
/* Warmup to force CPU caching */
while (i > 0 && !_currentRun.IsFaulted)
{
RunSingleBenchmark();
i--;
}
Trace.Debug("----- END WARMUPS -----");
}
else
{
Trace.Debug("----- SKIPPING WARMUPS -----");
}
}
/// <summary>
/// Creates an instance for all applicable <see cref="MetricCollector" />s for this metric type.
/// </summary>
/// <param name="runMode">
/// The <see cref="RunMode" /> for this benchmark. Influences the type of
/// <see cref="MetricCollector" /> used in some instances.
/// </param>
/// <param name="warmup">Warmup data. Influences the type of <see cref="MetricCollector" /> used in some instances.</param>
/// <param name="setting">An implementation-specific <see cref="IBenchmarkSetting" /></param>
/// <returns>A new <see cref="MetricCollector" /> instance.</returns>
public abstract MetricCollector Create(RunMode runMode, WarmupData warmup,
IBenchmarkSetting setting);
public bool Equals(WarmupData other)
{
return(ElapsedTicks == other.ElapsedTicks && ActualRunsMeasured == other.ActualRunsMeasured);
}
/// <summary>
/// Warmup phase
/// </summary>
private void WarmUp()
{
Trace.Debug($"Beginning Warmups for {BenchmarkName}");
var warmupStopWatch = new Stopwatch();
var targetTime = Settings.RunTime;
Contract.Assert(targetTime != TimeSpan.Zero);
var runCount = 0L;
var runTime = 0L;
/* Pre-Warmup */
Trace.Debug("----- BEGIN PRE-WARMUP -----");
/* Estimate */
Allocate(); // allocate all collectors needed
PreRun();
try
{
if (Settings.RunMode == RunMode.Throughput)
{
Trace.Debug(
$"Throughput mode: estimating how many invocations of {BenchmarkName} will take {targetTime.TotalSeconds}s");
var estimateCount = 3;
var runEstimates = new long[estimateCount];
var timeEstimates = new long[estimateCount];
for (var i = 0; i <= estimateCount; i++)
{
warmupStopWatch.Start();
while (warmupStopWatch.ElapsedTicks < targetTime.Ticks)
{
Invoker.InvokeRun(_currentRun.Context);
runCount++;
}
warmupStopWatch.Stop();
if (i > 0)
{
runEstimates[i - 1] = runCount;
timeEstimates[i - 1] = warmupStopWatch.ElapsedTicks;
}
runCount = 0;
warmupStopWatch.Reset();
}
runCount = (long)Math.Ceiling(runEstimates.Average());
runTime = (long)Math.Ceiling(timeEstimates.Average());
Trace.Debug(
$"Throughput mode: executed {runCount} instances of {BenchmarkName} in roughly {targetTime.TotalSeconds}s. Using that figure for benchmark.");
}
else
{
warmupStopWatch.Start();
Invoker.InvokeRun(_currentRun.Context);
runCount++;
warmupStopWatch.Stop();
// elapsed time
runTime = warmupStopWatch.ElapsedTicks;
}
}
catch (Exception ex)
{
HandleBenchmarkRunException(ex, $"Error occurred during ${BenchmarkName} RUN.");
}
PostRun();
Complete(true);
// check to see if pre-warmup threw an exception
var faulted = _currentRun.IsFaulted;
if (faulted)
{
Trace.Error($"Error occurred during pre-warmup. Exiting and producing dump...");
/*
* Normally we don't ever queue up the warmup into the final stats, but we do it
* in failure cases so we can capture the exception thrown during warmup into
* the final report we're going to deliver to the end-user.
*/
CompletedRuns.Enqueue(_currentRun.ToReport(TimeSpan.Zero));
return;
}
Trace.Debug("----- END PRE-WARMUP -----");
WarmupData = new WarmupData(runTime, runCount);
if (!Settings.SkipWarmups)
{
Trace.Debug("----- BEGIN WARMUPS -----");
var i = _warmupCount;
/* Warmup to force CPU caching */
while (i > 0 && !_currentRun.IsFaulted)
{
RunSingleBenchmark();
i--;
}
Trace.Debug("----- END WARMUPS -----");
}
else
{
Trace.Debug("----- SKIPPING WARMUPS -----");
}
}
/// <summary>
/// Creates an instance for all applicable <see cref="MetricCollector" />s for this metric type.
/// </summary>
/// <param name="runMode">
/// The <see cref="RunMode" /> for this benchmark. Influences the type of
/// <see cref="MetricCollector" /> used in some instances.
/// </param>
/// <param name="warmup">Warmup data. Influences the type of <see cref="MetricCollector" /> used in some instances.</param>
/// <param name="setting">An implementation-specific <see cref="IBenchmarkSetting" /></param>
/// <returns>At least 1 new <see cref="MetricCollector"/> instance. Each instance will be uniquely named.</returns>
public abstract IEnumerable<MetricCollector> Create(RunMode runMode, WarmupData warmup,
IBenchmarkSetting setting);