private static void RunTestCategory <ITree>( Kind run, Group group, IEnumerable <CreateTreeInfo <ITree> > creators, IEnumerable <TestInfo <ITree> > tests, ref bool success, bool resetBaseline, Dictionary <string, Measurement.Result> baselineResults, List <Measurement.Result> results) { foreach (CreateTreeInfo <ITree> createTreeInfo in creators) { if (createTreeInfo.group <= group) { foreach (TestInfo <ITree> test in tests) { Measurement.Result result = Measurement.RunTest( String.Format("{0}:{1}", createTreeInfo.kind, test.label), delegate() { return(test.makePerfTestFactory(createTreeInfo.treeFactory, test.count)()); }, run == Kind.Real ? test.trials : 1, test.multiplier); if (run == Kind.Real) { Measurement.Result baseline; baselineResults.TryGetValue(result.label, out baseline); bool success1 = DisplayResult(result, baseline) && ((baseline != null) || resetBaseline); success = success1 && (success || resetBaseline); } results.Add(result); } } } }
private static void TestAverage(Measurement.Result result, Measurement.Result baseline, out bool accepted, out bool improved) { if (baseline == null) { accepted = true; improved = false; return; } Test(result.average, baseline.average, MedianMeanEpsilon, out accepted, out improved); }
private static void TestStability(Measurement.Result result, Measurement.Result baseline, out bool accepted) { if (result.stability > StabilityEpsilon) { accepted = false; return; } if ((baseline != null) && (baseline.stability > StabilityEpsilon)) { accepted = false; return; } accepted = true; }
private static bool DisplayResult(Measurement.Result result, Measurement.Result baseline) { bool acceptMedian, acceptAverage, acceptStability; bool improvedMedian, improvedAverage; TestMedian(result, baseline, out acceptMedian, out improvedMedian); TestAverage(result, baseline, out acceptAverage, out improvedAverage); TestStability(result, baseline, out acceptStability); bool accepted = acceptMedian && acceptAverage && acceptStability; bool improved = improvedMedian || improvedAverage; DisplayLine( result.label, "med", acceptMedian ? (improvedMedian ? Theme.Green : Theme.Normal) : Theme.Red, result.median, "avg", acceptAverage ? (improvedAverage ? Theme.Green : Theme.Normal) : Theme.Red, result.average, "sta", acceptStability ? Theme.Normal : Theme.Red, result.stability, baseline != null ? Theme.Normal : Theme.Red, baseline != null ? String.Empty : "NB"); if (baseline != null) { DisplayLine( String.Empty, "x", acceptMedian ? (improvedMedian ? Theme.Green : Theme.Normal) : Theme.Red, result.median / baseline.median, "x", acceptAverage ? (improvedAverage ? Theme.Green : Theme.Normal) : Theme.Red, result.average / baseline.average, String.Empty /*"x"*/, Theme.Normal /*acceptStability ? Theme.Normal : Theme.Red*/, null /*result.stability / baseline.stability*/, Theme.Normal, String.Empty); } return(accepted); }
public static bool RunAllPerfTests(bool enabled, bool resetBaseline, Group group, KeyValuePair <string, bool>[] enables) { Debug.Assert(enables.Length == CategoryTokens.Length); bool success = false; string status = "SKIPPED"; Console.WriteLine("Performance Regression Tests - Started"); #if DEBUG #pragma warning disable CS0162 // complaint about unreachable code Console.WriteLine(" DEBUG build - skipping"); if (false) #else if (enabled) #endif { GCLatencyMode oldGCLatencyMode = GCSettings.LatencyMode; ProcessPriorityClass oldProcessPriority = Process.GetCurrentProcess().PriorityClass; ThreadPriority oldThreadPriority = Thread.CurrentThread.Priority; try { GCSettings.LatencyMode = GCLatencyMode.SustainedLowLatency; Process.GetCurrentProcess().PriorityClass = ProcessPriorityClass.AboveNormal; Thread.CurrentThread.Priority = ThreadPriority.AboveNormal; string baselinePath = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), "TreeLibTest", "perfbaseline.txt"); List <Measurement.Result> results = new List <Measurement.Result>(); success = true; Dictionary <string, Measurement.Result> baselineResults = new Dictionary <string, Measurement.Result>(); if (!resetBaseline) { if (File.Exists(baselinePath)) { using (TextReader reader = new StreamReader(baselinePath)) { string line = reader.ReadLine(); int version = Int32.Parse(line); if (version == Measurement.Result.Version) { reader.ReadLine(); // headers while ((line = reader.ReadLine()) != null) { Measurement.Result result = Measurement.Result.FromString(line); baselineResults.Add(result.label, result); } } else { Console.WriteLine(" ** baseline file has wrong version - not loaded **"); } } } else { Console.WriteLine(" ** no baseline file found **"); } } foreach (Kind phase in new Kind[] { Kind.Dry, Kind.Real }) { results.Clear(); TextWriter savedOutput = null; if (phase == Kind.Dry) { savedOutput = Console.Out; Console.SetOut(TextWriter.Null); } if (enables[Array.FindIndex(enables, delegate(KeyValuePair <string, bool> candidate) { return(String.Equals("basic", candidate.Key)); })].Value) { RunTestCategory(phase, group, MapCreators, MapTests, ref success, resetBaseline, baselineResults, results); RunTestCategory(phase, group, RankMapCreators, RankMapTests, ref success, resetBaseline, baselineResults, results); RunTestCategory(phase, group, MultiRankMapCreators, MultiRankMapTests, ref success, resetBaseline, baselineResults, results); RunTestCategory(phase, group, RangeMapCreators, RangeMapTests, ref success, resetBaseline, baselineResults, results); RunTestCategory(phase, group, Range2MapCreators, Range2MapTests, ref success, resetBaseline, baselineResults, results); } if (enables[Array.FindIndex(enables, delegate(KeyValuePair <string, bool> candidate) { return(String.Equals("enum", candidate.Key)); })].Value) { RunTestCategory(phase, group, MapEnumerationCreators, MapEnumerationTests, ref success, resetBaseline, baselineResults, results); RunTestCategory(phase, group, Range2MapEnumerationCreators, Range2MapEnumerationTests, ref success, resetBaseline, baselineResults, results); } if (savedOutput != null) { Console.SetOut(savedOutput); } } if (resetBaseline) { status = "NEW BASELINE"; Directory.CreateDirectory(Path.GetDirectoryName(baselinePath)); using (TextWriter writer = new StreamWriter(baselinePath)) { writer.WriteLine(Measurement.Result.Version); writer.WriteLine(Measurement.Result.Header); for (int i = 0; i < results.Count; i++) { writer.WriteLine(results[i].ToString()); } } } else { status = success ? "PASSED" : "FAILED"; } } finally { GCSettings.LatencyMode = oldGCLatencyMode; Process.GetCurrentProcess().PriorityClass = oldProcessPriority; Thread.CurrentThread.Priority = oldThreadPriority; } } Program.WritePassFail("Performance Regression Tests - Finished", success ? Program.TestResultCode.Passed : Program.TestResultCode.Failed, status); return(success); }