// Run benchmark - actually run benchmark the specified number of times and using the specified // controls and return results. public Results RunBenchmark(Benchmark benchmark) { bool doRun = Controls.DoRun; int numberOfRuns = Controls.NumberOfRunsPerBenchmark; bool doWarmUpRun = Controls.DoWarmUpRun; string runner = Controls.Runner; bool doDebugBenchmark = Controls.DoDebugBenchmark; bool doVerbose = Controls.DoVerbose; string complusVersion = Controls.ComplusVersion; string benchmarksRootDirectory = Controls.BenchmarksRootDirectory; string benchmarkDirectory = System.IO.Path.Combine(benchmarksRootDirectory, benchmark.WorkingDirectory); bool doRunInShell = benchmark.DoRunInShell; bool useSSE = benchmark.UseSSE; bool useAVX = benchmark.UseAVX; int expectedResults = benchmark.ExpectedResults; int failureResults = ~expectedResults; int actualResults; int failures = 0; Results results = new Results(benchmark, numberOfRuns); if (!doRun) { return results; } string workingDirectory = benchmarkDirectory; string fileName = System.IO.Path.Combine(workingDirectory, benchmark.ExeName); string args = benchmark.ExeArgs; if (runner != "") { args = fileName + " " + args; fileName = runner; } if (doDebugBenchmark) { args = "/debugexe " + fileName + " " + args; fileName = "devenv.exe"; } if (doRunInShell) { args = "/C " + fileName + " " + args; fileName = "cmd.exe"; } if (doVerbose) { Console.WriteLine("Running benchmark {0} ...", benchmark.Name); Console.WriteLine("Invoking: {0} {1}", fileName, args); } for (int run = (doWarmUpRun) ? 0 : 1; run <= numberOfRuns; run++) { ProcessStartInfo startInfo = new ProcessStartInfo() { FileName = fileName, Arguments = args, WorkingDirectory = workingDirectory, UseShellExecute = false }; if (complusVersion != "") { startInfo.Environment["COMPlus_Version"] = complusVersion; startInfo.Environment["COMPlus_DefaultVersion"] = complusVersion; } if (useSSE) { startInfo.Environment["COMPlus_FeatureSIMD"] = "1"; startInfo.Environment["COMPlus_EnableAVX"] = "0"; } if (useAVX) { startInfo.Environment["COMPlus_FeatureSIMD"] = "1"; startInfo.Environment["COMPlus_EnableAVX"] = "1"; } startInfo.Environment["COMPlus_gcConcurrent"] = "0"; startInfo.Environment["COMPlus_gcServer"] = "0"; startInfo.Environment["COMPlus_NoGuiOnAssert"] = "1"; startInfo.Environment["COMPlus_BreakOnUncaughtException"] = "0"; var clockTime = Stopwatch.StartNew(); int exitCode = 0; try { using (var proc = Process.Start(startInfo)) { proc.EnableRaisingEvents = true; proc.WaitForExit(); exitCode = proc.ExitCode; } this.NumberOfBenchmarksRun++; } catch (Exception exception) { Console.WriteLine("Could not launch test {0} exception: {1}", startInfo.FileName, exception); exitCode = failureResults; } clockTime.Stop(); actualResults = exitCode; long time = clockTime.ElapsedMilliseconds; if (actualResults != expectedResults) { failures++; time = 0; } results.Times[run] = time; if (doVerbose) { Console.Write("Iteration benchmark {0} ", benchmark.Name); if (actualResults == expectedResults) { Console.Write("elapsed time {0}ms", time); } else { Console.Write("FAILED(expected={0}, actual={1})", expectedResults, actualResults); } if (run == 0) { Console.Write(" (warmup)"); } Console.WriteLine(""); } } // Calculate min, max, avg, and std devation. long sum = 0; long minimum = results.Times[1]; long maximum = minimum; for (int run = 1; run <= numberOfRuns; run++) { long time = results.Times[run]; sum += time; minimum = Math.Min(minimum, time); maximum = Math.Max(maximum, time); } long average = sum / (long)numberOfRuns; double standardDeviation = 0.0; if (numberOfRuns > 1) { double s = 0.0; double a = (double)average; double n = (double)numberOfRuns; for (int run = 1; run <= numberOfRuns; run++) { double time = (double)results.Times[run]; double t = (time - a); s += (t * t); } double variance = s / n; if (a == 0.0) { standardDeviation = 0.0; } else { standardDeviation = 100.0 * (Math.Sqrt(variance) / a); // stddev as a percentage standardDeviation = Math.Round(standardDeviation, 2, MidpointRounding.AwayFromZero); } } // Record results and return. results.Average = average; results.Minimum = minimum; results.Maximum = maximum; results.StandardDeviation = standardDeviation; results.Failures = failures; return results; }
public Results ( Benchmark benchmark, int numberOfRuns ) { Benchmark = benchmark; Times = new long[numberOfRuns + 1]; // leave empty slot at index 0, not used. Minimum = 0; Maximum = 0; Average = 0; StandardDeviation = 0.0; Failures = 0; }
// Add a benchmark to the list of benchmarks read in from the .XML file. public void AddBenchmark ( string name, string suiteName, string tags, string workingDirectory, string exeName, string exeArgs, bool doRunInShell, bool useSSE, bool useAVX, int expectedResults ) { BenchmarkSuite benchmarkSuite; BenchmarkTagSet benchmarkTagSet; Benchmark benchmark; benchmark = new Benchmark(name, suiteName, tags, workingDirectory, exeName, exeArgs, doRunInShell, useSSE, useAVX, expectedResults); BenchmarkList.Add(benchmark); if (!BenchmarkSuiteTable.TryGetValue(suiteName, out benchmarkSuite)) { benchmarkSuite = new BenchmarkSuite(suiteName); BenchmarkSuiteTable.Add(suiteName, benchmarkSuite); } benchmarkSuite.BenchmarkList.Add(benchmark); string[] tagList = tags.Split(ListSeparatorCharSet, StringSplitOptions.RemoveEmptyEntries); foreach (string tag in tagList) { if (!BenchmarkTagSetTable.TryGetValue(tag, out benchmarkTagSet)) { benchmarkTagSet = new BenchmarkTagSet(tag); BenchmarkTagSetTable.Add(tag, benchmarkTagSet); } benchmarkTagSet.BenchmarkList.Add(benchmark); } }
// Run benchmark - actually run benchmark the specified number of times and using the specified // controls and return results. public Results RunBenchmark(Benchmark benchmark) { bool doRun = Controls.DoRun; int numberOfRuns = Controls.NumberOfRunsPerBenchmark; bool doWarmUpRun = Controls.DoWarmUpRun; string runner = Controls.Runner; bool doDebugBenchmark = Controls.DoDebugBenchmark; bool doVerbose = Controls.DoVerbose; string complusVersion = Controls.ComplusVersion; string benchmarksRootDirectory = Controls.BenchmarksRootDirectory; string benchmarkDirectory = System.IO.Path.Combine(benchmarksRootDirectory, benchmark.WorkingDirectory); bool doRunInShell = benchmark.DoRunInShell; bool useSSE = benchmark.UseSSE; bool useAVX = benchmark.UseAVX; int expectedResults = benchmark.ExpectedResults; int failureResults = ~expectedResults; int actualResults; int failures = 0; Results results = new Results(benchmark, numberOfRuns); if (!doRun) { return(results); } string workingDirectory = benchmarkDirectory; string fileName = System.IO.Path.Combine(workingDirectory, benchmark.ExeName); string args = benchmark.ExeArgs; if (runner != "") { args = fileName + " " + args; fileName = runner; } if (doDebugBenchmark) { args = "/debugexe " + fileName + " " + args; fileName = "devenv.exe"; } if (doRunInShell) { args = "/C " + fileName + " " + args; fileName = "cmd.exe"; } if (doVerbose) { Console.WriteLine("Running benchmark {0} ...", benchmark.Name); Console.WriteLine("Invoking: {0} {1}", fileName, args); } for (int run = (doWarmUpRun) ? 0 : 1; run <= numberOfRuns; run++) { ProcessStartInfo startInfo = new ProcessStartInfo() { FileName = fileName, Arguments = args, WorkingDirectory = workingDirectory, UseShellExecute = false }; if (complusVersion != "") { startInfo.Environment["COMPlus_Version"] = complusVersion; startInfo.Environment["COMPlus_DefaultVersion"] = complusVersion; } if (useSSE) { startInfo.Environment["COMPlus_FeatureSIMD"] = "1"; startInfo.Environment["COMPlus_EnableAVX"] = "0"; } if (useAVX) { startInfo.Environment["COMPlus_FeatureSIMD"] = "1"; startInfo.Environment["COMPlus_EnableAVX"] = "1"; } startInfo.Environment["COMPlus_gcConcurrent"] = "0"; startInfo.Environment["COMPlus_gcServer"] = "0"; startInfo.Environment["COMPlus_NoGuiOnAssert"] = "1"; startInfo.Environment["COMPlus_BreakOnUncaughtException"] = "0"; var clockTime = Stopwatch.StartNew(); int exitCode = 0; try { using (var proc = Process.Start(startInfo)) { proc.EnableRaisingEvents = true; proc.WaitForExit(); exitCode = proc.ExitCode; } this.NumberOfBenchmarksRun++; } catch (Exception exception) { Console.WriteLine("Could not launch test {0} exception: {1}", startInfo.FileName, exception); exitCode = failureResults; } clockTime.Stop(); actualResults = exitCode; long time = clockTime.ElapsedMilliseconds; if (actualResults != expectedResults) { failures++; time = 0; } results.Times[run] = time; if (doVerbose) { Console.Write("Iteration benchmark {0} ", benchmark.Name); if (actualResults == expectedResults) { Console.Write("elapsed time {0}ms", time); } else { Console.Write("FAILED(expected={0}, actual={1})", expectedResults, actualResults); } if (run == 0) { Console.Write(" (warmup)"); } Console.WriteLine(""); } } // Calculate min, max, avg, and std devation. long sum = 0; long minimum = results.Times[1]; long maximum = minimum; for (int run = 1; run <= numberOfRuns; run++) { long time = results.Times[run]; sum += time; minimum = Math.Min(minimum, time); maximum = Math.Max(maximum, time); } long average = sum / (long)numberOfRuns; double standardDeviation = 0.0; if (numberOfRuns > 1) { double s = 0.0; double a = (double)average; double n = (double)numberOfRuns; for (int run = 1; run <= numberOfRuns; run++) { double time = (double)results.Times[run]; double t = (time - a); s += (t * t); } double variance = s / n; if (a == 0.0) { standardDeviation = 0.0; } else { standardDeviation = 100.0 * (Math.Sqrt(variance) / a); // stddev as a percentage standardDeviation = Math.Round(standardDeviation, 2, MidpointRounding.AwayFromZero); } } // Record results and return. results.Average = average; results.Minimum = minimum; results.Maximum = maximum; results.StandardDeviation = standardDeviation; results.Failures = failures; return(results); }