private static void RunTestCategory <ITree>(
Kind run,
Group group,
IEnumerable <CreateTreeInfo <ITree> > creators,
IEnumerable <TestInfo <ITree> > tests,
ref bool success,
bool resetBaseline,
Dictionary <string, Measurement.Result> baselineResults,
List <Measurement.Result> results)
{
foreach (CreateTreeInfo <ITree> createTreeInfo in creators)
{
if (createTreeInfo.group <= group)
{
foreach (TestInfo <ITree> test in tests)
{
Measurement.Result result = Measurement.RunTest(
String.Format("{0}:{1}", createTreeInfo.kind, test.label),
delegate() { return(test.makePerfTestFactory(createTreeInfo.treeFactory, test.count)()); },
run == Kind.Real ? test.trials : 1,
test.multiplier);
if (run == Kind.Real)
{
Measurement.Result baseline;
baselineResults.TryGetValue(result.label, out baseline);
bool success1 = DisplayResult(result, baseline) && ((baseline != null) || resetBaseline);
success = success1 && (success || resetBaseline);
}
results.Add(result);
}
}
}
}
private static void TestAverage(Measurement.Result result, Measurement.Result baseline, out bool accepted, out bool improved)
{
if (baseline == null)
{
accepted = true;
improved = false;
return;
}
Test(result.average, baseline.average, MedianMeanEpsilon, out accepted, out improved);
}
private static void TestStability(Measurement.Result result, Measurement.Result baseline, out bool accepted)
{
if (result.stability > StabilityEpsilon)
{
accepted = false;
return;
}
if ((baseline != null) && (baseline.stability > StabilityEpsilon))
{
accepted = false;
return;
}
accepted = true;
}
private static bool DisplayResult(Measurement.Result result, Measurement.Result baseline)
{
bool acceptMedian, acceptAverage, acceptStability;
bool improvedMedian, improvedAverage;
TestMedian(result, baseline, out acceptMedian, out improvedMedian);
TestAverage(result, baseline, out acceptAverage, out improvedAverage);
TestStability(result, baseline, out acceptStability);
bool accepted = acceptMedian && acceptAverage && acceptStability;
bool improved = improvedMedian || improvedAverage;
DisplayLine(
result.label,
"med",
acceptMedian ? (improvedMedian ? Theme.Green : Theme.Normal) : Theme.Red,
result.median,
"avg",
acceptAverage ? (improvedAverage ? Theme.Green : Theme.Normal) : Theme.Red,
result.average,
"sta",
acceptStability ? Theme.Normal : Theme.Red,
result.stability,
baseline != null ? Theme.Normal : Theme.Red,
baseline != null ? String.Empty : "NB");
if (baseline != null)
{
DisplayLine(
String.Empty,
"x",
acceptMedian ? (improvedMedian ? Theme.Green : Theme.Normal) : Theme.Red,
result.median / baseline.median,
"x",
acceptAverage ? (improvedAverage ? Theme.Green : Theme.Normal) : Theme.Red,
result.average / baseline.average,
String.Empty /*"x"*/,
Theme.Normal /*acceptStability ? Theme.Normal : Theme.Red*/,
null /*result.stability / baseline.stability*/,
Theme.Normal,
String.Empty);
}
return(accepted);
}
public static bool RunAllPerfTests(bool enabled, bool resetBaseline, Group group, KeyValuePair <string, bool>[] enables)
{
Debug.Assert(enables.Length == CategoryTokens.Length);
bool success = false;
string status = "SKIPPED";
Console.WriteLine("Performance Regression Tests - Started");
#if DEBUG
#pragma warning disable CS0162 // complaint about unreachable code
Console.WriteLine(" DEBUG build - skipping");
if (false)
#else
if (enabled)
#endif
{
GCLatencyMode oldGCLatencyMode = GCSettings.LatencyMode;
ProcessPriorityClass oldProcessPriority = Process.GetCurrentProcess().PriorityClass;
ThreadPriority oldThreadPriority = Thread.CurrentThread.Priority;
try
{
GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency;
Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.AboveNormal;
Thread.CurrentThread.Priority = ThreadPriority.AboveNormal;
string baselinePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "TreeLibTest", "perfbaseline.txt");
List <Measurement.Result> results = new List <Measurement.Result>();
success = true;
Dictionary <string, Measurement.Result> baselineResults = new Dictionary <string, Measurement.Result>();
if (!resetBaseline)
{
if (File.Exists(baselinePath))
{
using (TextReader reader = new StreamReader(baselinePath))
{
string line = reader.ReadLine();
int version = Int32.Parse(line);
if (version == Measurement.Result.Version)
{
reader.ReadLine(); // headers
while ((line = reader.ReadLine()) != null)
{
Measurement.Result result = Measurement.Result.FromString(line);
baselineResults.Add(result.label, result);
}
}
else
{
Console.WriteLine(" ** baseline file has wrong version - not loaded **");
}
}
}
else
{
Console.WriteLine(" ** no baseline file found **");
}
}
foreach (Kind phase in new Kind[] { Kind.Dry, Kind.Real })
{
results.Clear();
TextWriter savedOutput = null;
if (phase == Kind.Dry)
{
savedOutput = Console.Out;
Console.SetOut(TextWriter.Null);
}
if (enables[Array.FindIndex(enables, delegate(KeyValuePair <string, bool> candidate) { return(String.Equals("basic", candidate.Key)); })].Value)
{
RunTestCategory(phase, group, MapCreators, MapTests, ref success, resetBaseline, baselineResults, results);
RunTestCategory(phase, group, RankMapCreators, RankMapTests, ref success, resetBaseline, baselineResults, results);
RunTestCategory(phase, group, MultiRankMapCreators, MultiRankMapTests, ref success, resetBaseline, baselineResults, results);
RunTestCategory(phase, group, RangeMapCreators, RangeMapTests, ref success, resetBaseline, baselineResults, results);
RunTestCategory(phase, group, Range2MapCreators, Range2MapTests, ref success, resetBaseline, baselineResults, results);
}
if (enables[Array.FindIndex(enables, delegate(KeyValuePair <string, bool> candidate) { return(String.Equals("enum", candidate.Key)); })].Value)
{
RunTestCategory(phase, group, MapEnumerationCreators, MapEnumerationTests, ref success, resetBaseline, baselineResults, results);
RunTestCategory(phase, group, Range2MapEnumerationCreators, Range2MapEnumerationTests, ref success, resetBaseline, baselineResults, results);
}
if (savedOutput != null)
{
Console.SetOut(savedOutput);
}
}
if (resetBaseline)
{
status = "NEW BASELINE";
Directory.CreateDirectory(Path.GetDirectoryName(baselinePath));
using (TextWriter writer = new StreamWriter(baselinePath))
{
writer.WriteLine(Measurement.Result.Version);
writer.WriteLine(Measurement.Result.Header);
for (int i = 0; i < results.Count; i++)
{
writer.WriteLine(results[i].ToString());
}
}
}
else
{
status = success ? "PASSED" : "FAILED";
}
}
finally
{
GCSettings.LatencyMode = oldGCLatencyMode;
Process.GetCurrentProcess().PriorityClass = oldProcessPriority;
Thread.CurrentThread.Priority = oldThreadPriority;
}
}
Program.WritePassFail("Performance Regression Tests - Finished", success ? Program.TestResultCode.Passed : Program.TestResultCode.Failed, status);
return(success);
}