static void Benchmark(object obj, uint iterations, string suffix = null)
{
var bench = new BenchShark(true);
var result = bench.EvaluateDecoratedTasks(obj, iterations);
var results = result.FastestEvaluations.Select(x =>
{
var series = x.Iterations.Select(it => (double)it.ElapsedTicks).ToArray();
Array.Sort(series);
var summary = SortedArrayStatistics.FiveNumberSummary(series);
var ms = ArrayStatistics.MeanStandardDeviation(series);
return(new { x.Name, Mean = ms.Item1, StdDev = ms.Item2, Min = summary[0], Q1 = summary[1], Median = summary[2], Q3 = summary[3], Max = summary[4] });
}).ToArray();
var top = results[0];
var managed = results.Single(x => x.Name.StartsWith("Managed"));
var label = string.IsNullOrEmpty(suffix) ? obj.GetType().FullName : string.Concat(obj.GetType().FullName, ": ", suffix);
results.Select(x => new
{
x.Name,
Mean = Math.Round(x.Mean), StdDev = Math.Round(x.StdDev),
Min = Math.Round(x.Min), Q1 = Math.Round(x.Q1), Median = Math.Round(x.Median), Q3 = Math.Round(x.Q3), Max = Math.Round(x.Max),
TopSlowdown = Math.Round(x.Median / top.Median, 2),
ManagedSpeedup = Math.Round(managed.Median / x.Median, 2)
}).Dump(label);
}
public void EvaluateTask_TenIterations_WithEvaluationEvent()
{
// Arrange
var shark = new BenchShark {
EnableUnoptimizedEvaluations = true
};
var counter = 0;
var passed = false;
shark.EvaluationCompleted += (sender, args) =>
{
counter++;
Assert.IsTrue(args.TaskEvaluated.TotalElapsedTicks > args.TaskEvaluated.TotalExecutionTime.TotalMilliseconds);
Assert.AreEqual(null, args.TaskEvaluated.Name);
Assert.IsTrue(args.TaskEvaluated.AverageElapsedTicks < args.TaskEvaluated.TotalElapsedTicks);
Assert.IsTrue(args.TaskEvaluated.AverageExecutionTime < args.TaskEvaluated.TotalExecutionTime);
Assert.IsTrue(args.TaskEvaluated.WorstExecutionTime > args.TaskEvaluated.BestExecutionTime);
Assert.IsTrue(args.TaskEvaluated.WorstElapsedTicks > args.TaskEvaluated.BestElapsedTicks);
passed = true;
};
// Act
shark.EvaluateTask(TaskToEvaluate, 10);
// Assert
Assert.AreEqual(1, counter);
Assert.IsTrue(passed);
}
public void EvaluateTask_TenIterations_WithIterationEvent()
{
// Arrange
var shark = new BenchShark {
EnableUnoptimizedEvaluations = true
};
var counter = 0;
var passed = false;
shark.IterationCompleted += (sender, args) =>
{
// Assert the current iteration
Assert.IsTrue(args.CurrentIteration.ElapsedTicks > args.CurrentIteration.ExecutionTime.TotalMilliseconds);
// Assert the current evaluation
Assert.AreEqual(++counter, args.CurrentEvaluation.IterationsCount);
Assert.AreEqual(null, args.CurrentEvaluation.Name);
// The following is true when more than one iteration is performed
if (args.CurrentEvaluation.IterationsCount > 1)
{
Assert.IsTrue(args.CurrentEvaluation.AverageElapsedTicks < args.CurrentEvaluation.TotalElapsedTicks);
Assert.IsTrue(args.CurrentEvaluation.AverageExecutionTime < args.CurrentEvaluation.TotalExecutionTime);
Assert.IsTrue(args.CurrentEvaluation.WorstExecutionTime > args.CurrentEvaluation.BestExecutionTime);
Assert.IsTrue(args.CurrentEvaluation.WorstElapsedTicks > args.CurrentEvaluation.BestElapsedTicks);
passed = true;
}
};
// Act
var result = shark.EvaluateTask(TaskToEvaluate, 10);
// Assert
Assert.AreEqual(10, result.IterationsCount);
Assert.IsTrue(passed);
}
public void EvaluateStoredTasks_TwoTasks_TenIteration_WithEvaluationEvent()
{
// Arrange
var shark = new BenchShark {
EnableUnoptimizedEvaluations = true
};
var names = new[] { "Foo", "Bar" };
var counter = 0;
shark.EvaluationCompleted += (sender, args) =>
{
Assert.AreEqual(names[counter], args.TaskEvaluated.Name);
Assert.IsTrue(args.TaskEvaluated.AverageElapsedTicks < args.TaskEvaluated.TotalElapsedTicks);
Assert.IsTrue(args.TaskEvaluated.AverageExecutionTime < args.TaskEvaluated.TotalExecutionTime);
Assert.AreEqual(10, args.TaskEvaluated.IterationsCount);
Assert.IsTrue(args.TaskEvaluated.WorstExecutionTime > args.TaskEvaluated.BestExecutionTime);
Assert.IsTrue(args.TaskEvaluated.WorstElapsedTicks > args.TaskEvaluated.BestElapsedTicks);
counter++;
};
// Act
shark.AddTask(names[0], TaskToEvaluate1);
shark.AddTask(names[1], TaskToEvaluate2);
shark.EvaluateStoredTasks(10);
// Assert
Assert.AreEqual(2, counter);
}
public void EvaluateStoredTasks_OneTask_TenIteration()
{
// Arrange
var shark = new BenchShark {
EnableUnoptimizedEvaluations = true
};
const string name = "Foo";
// Act
shark.AddTask(name, TaskToEvaluate1);
var allResults = shark.EvaluateStoredTasks(10);
var result = allResults.Evaluations.First();
// Assert collection
Assert.AreEqual(1, allResults.Evaluations.Count());
CollectionAssert.AreEqual(allResults.FastestEvaluations.ToArray(), allResults.SlowestEvaluations.ToArray());
// Assert evaluation
Assert.IsTrue(result.AverageElapsedTicks < result.TotalElapsedTicks);
Assert.IsTrue(result.AverageExecutionTime < result.TotalExecutionTime);
Assert.AreEqual(10, result.IterationsCount);
Assert.AreEqual(name, result.Name);
Assert.IsTrue(result.WorstExecutionTime > result.BestExecutionTime);
Assert.IsTrue(result.WorstElapsedTicks > result.BestElapsedTicks);
}
public void EvaluateTask_OneIteration_WithEvaluationEvent()
{
// Arrange
var shark = new BenchShark(true) {EnableUnoptimizedEvaluations = true};
var passed = false;
shark.EvaluationCompleted += (sender, args) =>
{
// Assert
Assert.AreNotEqual(0, args.TaskEvaluated.TotalElapsedTicks);
Assert.AreNotEqual(TimeSpan.Zero, args.TaskEvaluated.TotalExecutionTime);
Assert.IsTrue(args.TaskEvaluated.TotalElapsedTicks > args.TaskEvaluated.TotalExecutionTime.TotalMilliseconds);
Assert.AreEqual(1, args.TaskEvaluated.IterationsCount);
Assert.AreEqual(1, args.TaskEvaluated.Iterations.Count());
Assert.AreEqual(args.TaskEvaluated.TotalElapsedTicks, args.TaskEvaluated.AverageElapsedTicks);
Assert.AreEqual(args.TaskEvaluated.TotalExecutionTime, args.TaskEvaluated.AverageExecutionTime);
Assert.AreEqual(args.TaskEvaluated.TotalElapsedTicks, args.TaskEvaluated.BestElapsedTicks);
Assert.AreEqual(args.TaskEvaluated.TotalElapsedTicks, args.TaskEvaluated.WorstElapsedTicks);
Assert.AreEqual(args.TaskEvaluated.TotalExecutionTime, args.TaskEvaluated.BestExecutionTime);
Assert.AreEqual(args.TaskEvaluated.TotalExecutionTime, args.TaskEvaluated.WorstExecutionTime);
Assert.AreEqual(args.TaskEvaluated.TotalElapsedTicks, args.TaskEvaluated.TotalElapsedTicks);
Assert.AreEqual(args.TaskEvaluated.TotalExecutionTime, args.TaskEvaluated.TotalExecutionTime);
passed = true;
};
// Act
shark.EvaluateTask(TaskToEvaluate, 1);
// Assert
Assert.IsTrue(passed);
}
public void EvaluateStoredTasks_TwoTasks_OneIteration()
{
// Arrange
var shark = new BenchShark {
EnableUnoptimizedEvaluations = true
};
const string name1 = "Foo";
const string name2 = "Bar";
// Act
shark.AddTask(name1, TaskToEvaluate1);
shark.AddTask(name2, TaskToEvaluate2);
var allResults = shark.EvaluateStoredTasks(1);
// Assert collection
Assert.AreEqual(2, allResults.Evaluations.Count());
Assert.AreNotEqual(allResults.FastestEvaluations, allResults.SlowestEvaluations);
// Assert evaluation
Assert.AreEqual(name1, allResults.Evaluations.ElementAt(0).Name);
Assert.AreEqual(name2, allResults.Evaluations.ElementAt(1).Name);
foreach (var result in allResults.Evaluations)
{
Assert.AreEqual(result.AverageElapsedTicks, result.TotalElapsedTicks);
Assert.AreEqual(result.AverageExecutionTime, result.TotalExecutionTime);
Assert.AreEqual(1, result.IterationsCount);
Assert.IsTrue(result.WorstExecutionTime == result.BestExecutionTime);
Assert.IsTrue(result.WorstElapsedTicks == result.BestElapsedTicks);
}
}
public void EvaluateDecoratedTasks_Obj_TenIteration()
{
// Arrange
var shark = new BenchShark(true) { EnableUnoptimizedEvaluations = true };
SetupEnvironment();
// Act
var methods = shark.EvaluateDecoratedTasks(this, 10);
// Assert
Assert.AreEqual(3, methods.Evaluations.Count());
Assert.AreNotEqual(methods.Evaluations.ElementAt(0), methods.Evaluations.ElementAt(1));
Assert.AreEqual(11, FooRun); // one execution to jut the function, second to evaluate
Assert.AreEqual(11, BarRun);
Assert.AreEqual(11, StaticFoobarRun);
}
public void EvaluateTask_OneIteration()
{
// Arrange
var shark = new BenchShark(true) {EnableUnoptimizedEvaluations = true};
// Act
var result = shark.EvaluateTask(TaskToEvaluate, 1);
// Assert
Assert.AreEqual(1, result.IterationsCount);
Assert.AreEqual(1, result.Iterations.Count());
Assert.AreEqual(result.TotalElapsedTicks, result.AverageElapsedTicks);
Assert.AreEqual(result.TotalExecutionTime, result.AverageExecutionTime);
Assert.AreEqual(result.TotalElapsedTicks, result.BestElapsedTicks);
Assert.AreEqual(result.TotalElapsedTicks, result.WorstElapsedTicks);
Assert.AreEqual(result.TotalExecutionTime, result.BestExecutionTime);
Assert.AreEqual(result.TotalExecutionTime, result.WorstExecutionTime);
}
public void EvaluateDecoratedTasks_Type_TenIteration()
{
// Arrange
var shark = new BenchShark(true)
{
EnableUnoptimizedEvaluations = true
};
SetupEnvironment();
// Act
var methods = shark.EvaluateDecoratedTasks <ReflectionEvaluationTests>(10);
// Assert
Assert.AreEqual(3, methods.Evaluations.Count());
Assert.AreNotEqual(methods.Evaluations.ElementAt(0), methods.Evaluations.ElementAt(1));
Assert.AreEqual(11, FooRun);
Assert.AreEqual(11, BarRun);
Assert.AreEqual(11, StaticFoobarRun);
}
public void EvaluateTask_TenIterations()
{
// Arrange
var shark = new BenchShark(true)
{
EnableUnoptimizedEvaluations = true
};
// Act
var result = shark.EvaluateTask(TaskToEvaluate, 10);
// Assert
Assert.AreEqual(10, result.IterationsCount);
Assert.AreEqual(10, result.Iterations.Count());
Assert.IsTrue(result.AverageElapsedTicks < result.TotalElapsedTicks);
Assert.IsTrue(result.AverageExecutionTime < result.TotalExecutionTime);
Assert.AreEqual(null, result.Name);
Assert.IsTrue(result.WorstExecutionTime > result.BestExecutionTime);
Assert.IsTrue(result.WorstElapsedTicks > result.BestElapsedTicks);
}
public void EvaluateDecoratedTasks_Obj_OneIteration()
{
// Arrange
var shark = new BenchShark(true)
{
EnableUnoptimizedEvaluations = true
};
SetupEnvironment();
// Act
var methods = shark.EvaluateDecoratedTasks(this, 1);
// Assert
Assert.AreEqual(3, methods.Evaluations.Count());
Assert.AreNotEqual(methods.Evaluations.ElementAt(0), methods.Evaluations.ElementAt(1));
Assert.AreEqual(2, FooRun); // one execution to jut the function, second to evaluate
Assert.AreEqual(2, BarRun);
Assert.AreEqual(2, StaticFoobarRun);
}
public void EvaluateTask_OneIteration()
{
// Arrange
var shark = new BenchShark(true)
{
EnableUnoptimizedEvaluations = true
};
// Act
var result = shark.EvaluateTask(TaskToEvaluate, 1);
// Assert
Assert.AreEqual(1, result.IterationsCount);
Assert.AreEqual(1, result.Iterations.Count());
Assert.AreEqual(result.TotalElapsedTicks, result.AverageElapsedTicks);
Assert.AreEqual(result.TotalExecutionTime, result.AverageExecutionTime);
Assert.AreEqual(result.TotalElapsedTicks, result.BestElapsedTicks);
Assert.AreEqual(result.TotalElapsedTicks, result.WorstElapsedTicks);
Assert.AreEqual(result.TotalExecutionTime, result.BestExecutionTime);
Assert.AreEqual(result.TotalExecutionTime, result.WorstExecutionTime);
}
public void EvaluateStoredTasks_OneTask_TenIteration()
{
// Arrange
var shark = new BenchShark { EnableUnoptimizedEvaluations = true };
const string name = "Foo";
// Act
shark.AddTask(name, TaskToEvaluate1);
var allResults = shark.EvaluateStoredTasks(10);
var result = allResults.Evaluations.First();
// Assert collection
Assert.AreEqual(1, allResults.Evaluations.Count());
CollectionAssert.AreEqual(allResults.FastestEvaluations.ToArray(), allResults.SlowestEvaluations.ToArray());
// Assert evaluation
Assert.IsTrue(result.AverageElapsedTicks < result.TotalElapsedTicks);
Assert.IsTrue(result.AverageExecutionTime < result.TotalExecutionTime);
Assert.AreEqual(10, result.IterationsCount);
Assert.AreEqual(name, result.Name);
Assert.IsTrue(result.WorstExecutionTime > result.BestExecutionTime);
Assert.IsTrue(result.WorstElapsedTicks > result.BestElapsedTicks);
}
public void EvaluateTask_OneIteration_WithEvaluationEvent()
{
// Arrange
var shark = new BenchShark(true)
{
EnableUnoptimizedEvaluations = true
};
var passed = false;
shark.EvaluationCompleted += (sender, args) =>
{
// Assert
Assert.AreNotEqual(0, args.TaskEvaluated.TotalElapsedTicks);
Assert.AreNotEqual(TimeSpan.Zero, args.TaskEvaluated.TotalExecutionTime);
Assert.IsTrue(args.TaskEvaluated.TotalElapsedTicks > args.TaskEvaluated.TotalExecutionTime.TotalMilliseconds);
Assert.AreEqual(1, args.TaskEvaluated.IterationsCount);
Assert.AreEqual(1, args.TaskEvaluated.Iterations.Count());
Assert.AreEqual(args.TaskEvaluated.TotalElapsedTicks, args.TaskEvaluated.AverageElapsedTicks);
Assert.AreEqual(args.TaskEvaluated.TotalExecutionTime, args.TaskEvaluated.AverageExecutionTime);
Assert.AreEqual(args.TaskEvaluated.TotalElapsedTicks, args.TaskEvaluated.BestElapsedTicks);
Assert.AreEqual(args.TaskEvaluated.TotalElapsedTicks, args.TaskEvaluated.WorstElapsedTicks);
Assert.AreEqual(args.TaskEvaluated.TotalExecutionTime, args.TaskEvaluated.BestExecutionTime);
Assert.AreEqual(args.TaskEvaluated.TotalExecutionTime, args.TaskEvaluated.WorstExecutionTime);
Assert.AreEqual(args.TaskEvaluated.TotalElapsedTicks, args.TaskEvaluated.TotalElapsedTicks);
Assert.AreEqual(args.TaskEvaluated.TotalExecutionTime, args.TaskEvaluated.TotalExecutionTime);
passed = true;
};
// Act
shark.EvaluateTask(TaskToEvaluate, 1);
// Assert
Assert.IsTrue(passed);
}
static void Benchmark(object obj, uint iterations, string suffix = null)
{
var bench = new BenchShark(true);
var result = bench.EvaluateDecoratedTasks(obj, iterations);
var results = result.FastestEvaluations.Select(x =>
{
var series = x.Iterations.Select(it => (double)it.ElapsedTicks).ToArray();
Array.Sort(series);
var summary = SortedArrayStatistics.FiveNumberSummary(series);
var ms = ArrayStatistics.MeanStandardDeviation(series);
return new { x.Name, Mean = ms.Item1, StdDev = ms.Item2, Min = summary[0], Q1 = summary[1], Median = summary[2], Q3 = summary[3], Max = summary[4] };
}).ToArray();
var top = results[0];
var managed = results.Single(x => x.Name.StartsWith("Managed"));
var label = string.IsNullOrEmpty(suffix) ? obj.GetType().FullName : string.Concat(obj.GetType().FullName, ": ", suffix);
results.Select(x => new
{
x.Name,
Mean = Math.Round(x.Mean), StdDev = Math.Round(x.StdDev),
Min = Math.Round(x.Min), Q1 = Math.Round(x.Q1), Median = Math.Round(x.Median), Q3 = Math.Round(x.Q3), Max = Math.Round(x.Max),
TopSlowdown = Math.Round(x.Median/top.Median, 2),
ManagedSpeedup = Math.Round(managed.Median/x.Median, 2)
}).Dump(label);
}
public void EvaluateStoredTasks_TwoTasks_TenIteration_WithEvaluationEvent()
{
// Arrange
var shark = new BenchShark { EnableUnoptimizedEvaluations = true };
var names = new[] { "Foo", "Bar" };
var counter = 0;
shark.EvaluationCompleted += (sender, args) =>
{
Assert.AreEqual(names[counter], args.TaskEvaluated.Name);
Assert.IsTrue(args.TaskEvaluated.AverageElapsedTicks < args.TaskEvaluated.TotalElapsedTicks);
Assert.IsTrue(args.TaskEvaluated.AverageExecutionTime < args.TaskEvaluated.TotalExecutionTime);
Assert.AreEqual(10, args.TaskEvaluated.IterationsCount);
Assert.IsTrue(args.TaskEvaluated.WorstExecutionTime > args.TaskEvaluated.BestExecutionTime);
Assert.IsTrue(args.TaskEvaluated.WorstElapsedTicks > args.TaskEvaluated.BestElapsedTicks);
counter++;
};
// Act
shark.AddTask(names[0], TaskToEvaluate1);
shark.AddTask(names[1], TaskToEvaluate2);
shark.EvaluateStoredTasks(10);
// Assert
Assert.AreEqual(2, counter);
}
public void EvaluateStoredTasks_TwoTasks_OneIteration()
{
// Arrange
var shark = new BenchShark { EnableUnoptimizedEvaluations = true };
const string name1 = "Foo";
const string name2 = "Bar";
// Act
shark.AddTask(name1, TaskToEvaluate1);
shark.AddTask(name2, TaskToEvaluate2);
var allResults = shark.EvaluateStoredTasks(1);
// Assert collection
Assert.AreEqual(2, allResults.Evaluations.Count());
Assert.AreNotEqual(allResults.FastestEvaluations, allResults.SlowestEvaluations);
// Assert evaluation
Assert.AreEqual(name1, allResults.Evaluations.ElementAt(0).Name);
Assert.AreEqual(name2, allResults.Evaluations.ElementAt(1).Name);
foreach (var result in allResults.Evaluations)
{
Assert.AreEqual(result.AverageElapsedTicks, result.TotalElapsedTicks);
Assert.AreEqual(result.AverageExecutionTime, result.TotalExecutionTime);
Assert.AreEqual(1, result.IterationsCount);
Assert.IsTrue(result.WorstExecutionTime == result.BestExecutionTime);
Assert.IsTrue(result.WorstElapsedTicks == result.BestElapsedTicks);
}
}
public void EvaluateDecoratedTasks_Type_OneIteration()
{
// Arrange
var shark = new BenchShark(true) {EnableUnoptimizedEvaluations = true};
SetupEnvironment();
// Act
var methods = shark.EvaluateDecoratedTasks<ReflectionEvaluationTests>(1);
// Assert
Assert.AreEqual(3, methods.Evaluations.Count());
Assert.AreNotEqual(methods.Evaluations.ElementAt(0), methods.Evaluations.ElementAt(1));
Assert.AreEqual(2, FooRun);
Assert.AreEqual(2, BarRun);
Assert.AreEqual(2, StaticFoobarRun);
}
public void EvaluateTask_TenIterations()
{
// Arrange
var shark = new BenchShark(true) {EnableUnoptimizedEvaluations = true};
// Act
var result = shark.EvaluateTask(TaskToEvaluate, 10);
// Assert
Assert.AreEqual(10, result.IterationsCount);
Assert.AreEqual(10, result.Iterations.Count());
Assert.IsTrue(result.AverageElapsedTicks < result.TotalElapsedTicks);
Assert.IsTrue(result.AverageExecutionTime < result.TotalExecutionTime);
Assert.AreEqual(null, result.Name);
Assert.IsTrue(result.WorstExecutionTime > result.BestExecutionTime);
Assert.IsTrue(result.WorstElapsedTicks > result.BestElapsedTicks);
}
public void EvaluateTask_TenIterations_WithIterationEvent()
{
// Arrange
var shark = new BenchShark {EnableUnoptimizedEvaluations = true};
var counter = 0;
var passed = false;
shark.IterationCompleted += (sender, args) =>
{
// Assert the current iteration
Assert.IsTrue(args.CurrentIteration.ElapsedTicks > args.CurrentIteration.ExecutionTime.TotalMilliseconds);
// Assert the current evaluation
Assert.AreEqual(++counter, args.CurrentEvaluation.IterationsCount);
Assert.AreEqual(null, args.CurrentEvaluation.Name);
// The following is true when more than one iteration is performed
if (args.CurrentEvaluation.IterationsCount > 1)
{
Assert.IsTrue(args.CurrentEvaluation.AverageElapsedTicks < args.CurrentEvaluation.TotalElapsedTicks);
Assert.IsTrue(args.CurrentEvaluation.AverageExecutionTime < args.CurrentEvaluation.TotalExecutionTime);
Assert.IsTrue(args.CurrentEvaluation.WorstExecutionTime > args.CurrentEvaluation.BestExecutionTime);
Assert.IsTrue(args.CurrentEvaluation.WorstElapsedTicks > args.CurrentEvaluation.BestElapsedTicks);
passed = true;
}
};
// Act
var result = shark.EvaluateTask(TaskToEvaluate, 10);
// Assert
Assert.AreEqual(10, result.IterationsCount);
Assert.IsTrue(passed);
}
public void EvaluateTask_TenIterations_WithEvaluationEvent()
{
// Arrange
var shark = new BenchShark {EnableUnoptimizedEvaluations = true};
var counter = 0;
var passed = false;
shark.EvaluationCompleted += (sender, args) =>
{
counter++;
Assert.IsTrue(args.TaskEvaluated.TotalElapsedTicks > args.TaskEvaluated.TotalExecutionTime.TotalMilliseconds);
Assert.AreEqual(null, args.TaskEvaluated.Name);
Assert.IsTrue(args.TaskEvaluated.AverageElapsedTicks < args.TaskEvaluated.TotalElapsedTicks);
Assert.IsTrue(args.TaskEvaluated.AverageExecutionTime < args.TaskEvaluated.TotalExecutionTime);
Assert.IsTrue(args.TaskEvaluated.WorstExecutionTime > args.TaskEvaluated.BestExecutionTime);
Assert.IsTrue(args.TaskEvaluated.WorstElapsedTicks > args.TaskEvaluated.BestElapsedTicks);
passed = true;
};
// Act
shark.EvaluateTask(TaskToEvaluate, 10);
// Assert
Assert.AreEqual(1, counter);
Assert.IsTrue(passed);
}