private IEnumerable <PerformanceMetric> ConvertResultToMetrics(RunResult runResult)
{
var metrics = new List <PerformanceMetric>();
if (runResult.Successful)
{
metrics.Add(
new PerformanceMetric
{
Scenario = runResult.TestName,
Metric = "total",
Unit = "Milliseconds",
Value = runResult.ElapsedMillis
});
if (runResult.IterationCounters.Count > 1)
{
foreach (var i in new[] { 0.95, 0.99, 0.999 })
{
var percentile = (i * 100).ToString(CultureInfo.InvariantCulture);
var resultPercentile = GetPercentile(runResult, i);
var metric = string.Format("{0}th percentile", percentile);
metrics.Add(
new PerformanceMetric
{
Scenario = runResult.TestName,
Metric = metric,
Unit = "Milliseconds",
Value = resultPercentile
});
}
}
}
return(metrics);
}
protected RunResult Run(TestDefinition test)
{
//localize test settings
var warmupCount = 0;
if (test.WarmupCount.HasValue)
{
warmupCount = test.WarmupCount.Value;
}
var iterationCount = 100;
if (test.IterationCount.HasValue)
{
iterationCount = test.IterationCount.Value;
}
var testName = test.TestName ?? test.GetType() + "#" + test.GetHashCode();
var setup = test.Setup;
var run = test.Run;
var cleanup = test.Cleanup;
//validate
if (run == null)
{
throw new ArgumentNullException(string.Format("Verify that test {0} has a run action.", testName));
}
//setup
try
{
if (setup != null)
{
setup();
}
}
catch (Exception e)
{
return(new RunResult(testName, e));
}
//warmup
try
{
for (var w = 0; w < warmupCount; ++w)
{
run();
}
}
catch (Exception e)
{
return(new RunResult(testName, e));
}
var runStopwatch = new Stopwatch();
var iterationStopwatch = new Stopwatch();
var iterationCounters = new List <IterationCounter>();
//run
try
{
for (var i = 0; i < iterationCount; ++i)
{
iterationStopwatch.Restart();
runStopwatch.Start();
run();
runStopwatch.Stop();
iterationStopwatch.Stop();
iterationCounters.Add(
new IterationCounter
{
ElapsedMillis = iterationStopwatch.ElapsedMilliseconds,
WorkingSet = GC.GetTotalMemory(false)
});
}
}
catch (Exception e)
{
return(new RunResult(testName, e));
}
var result = new RunResult(testName, runStopwatch.ElapsedMilliseconds, GC.GetTotalMemory(false), iterationCounters);
//cleanup
try
{
if (cleanup != null)
{
cleanup();
}
}
catch (Exception e)
{
result.ReportedException = e;
}
//report
return(result);
}