// Run benchmark - actually run benchmark the specified number of times and using the specified
// controls and return results.
public Results RunBenchmark(Benchmark benchmark)
{
bool doRun = Controls.DoRun;
int numberOfRuns = Controls.NumberOfRunsPerBenchmark;
bool doWarmUpRun = Controls.DoWarmUpRun;
string runner = Controls.Runner;
bool doDebugBenchmark = Controls.DoDebugBenchmark;
bool doVerbose = Controls.DoVerbose;
string complusVersion = Controls.ComplusVersion;
string benchmarksRootDirectory = Controls.BenchmarksRootDirectory;
string benchmarkDirectory = System.IO.Path.Combine(benchmarksRootDirectory, benchmark.WorkingDirectory);
bool doRunInShell = benchmark.DoRunInShell;
bool useSSE = benchmark.UseSSE;
bool useAVX = benchmark.UseAVX;
int expectedResults = benchmark.ExpectedResults;
int failureResults = ~expectedResults;
int actualResults;
int failures = 0;
Results results = new Results(benchmark, numberOfRuns);
if (!doRun)
{
return results;
}
string workingDirectory = benchmarkDirectory;
string fileName = System.IO.Path.Combine(workingDirectory, benchmark.ExeName);
string args = benchmark.ExeArgs;
if (runner != "")
{
args = fileName + " " + args;
fileName = runner;
}
if (doDebugBenchmark)
{
args = "/debugexe " + fileName + " " + args;
fileName = "devenv.exe";
}
if (doRunInShell)
{
args = "/C " + fileName + " " + args;
fileName = "cmd.exe";
}
if (doVerbose)
{
Console.WriteLine("Running benchmark {0} ...", benchmark.Name);
Console.WriteLine("Invoking: {0} {1}", fileName, args);
}
for (int run = (doWarmUpRun) ? 0 : 1; run <= numberOfRuns; run++)
{
ProcessStartInfo startInfo = new ProcessStartInfo()
{
FileName = fileName,
Arguments = args,
WorkingDirectory = workingDirectory,
UseShellExecute = false
};
if (complusVersion != "")
{
startInfo.Environment["COMPlus_Version"] = complusVersion;
startInfo.Environment["COMPlus_DefaultVersion"] = complusVersion;
}
if (useSSE)
{
startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
startInfo.Environment["COMPlus_EnableAVX"] = "0";
}
if (useAVX)
{
startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
startInfo.Environment["COMPlus_EnableAVX"] = "1";
}
startInfo.Environment["COMPlus_gcConcurrent"] = "0";
startInfo.Environment["COMPlus_gcServer"] = "0";
startInfo.Environment["COMPlus_NoGuiOnAssert"] = "1";
startInfo.Environment["COMPlus_BreakOnUncaughtException"] = "0";
var clockTime = Stopwatch.StartNew();
int exitCode = 0;
try
{
using (var proc = Process.Start(startInfo))
{
proc.EnableRaisingEvents = true;
proc.WaitForExit();
exitCode = proc.ExitCode;
}
this.NumberOfBenchmarksRun++;
}
catch (Exception exception)
{
Console.WriteLine("Could not launch test {0} exception: {1}",
startInfo.FileName, exception);
exitCode = failureResults;
}
clockTime.Stop();
actualResults = exitCode;
long time = clockTime.ElapsedMilliseconds;
if (actualResults != expectedResults)
{
failures++;
time = 0;
}
results.Times[run] = time;
if (doVerbose)
{
Console.Write("Iteration benchmark {0} ", benchmark.Name);
if (actualResults == expectedResults)
{
Console.Write("elapsed time {0}ms", time);
}
else
{
Console.Write("FAILED(expected={0}, actual={1})", expectedResults, actualResults);
}
if (run == 0)
{
Console.Write(" (warmup)");
}
Console.WriteLine("");
}
}
// Calculate min, max, avg, and std devation.
long sum = 0;
long minimum = results.Times[1];
long maximum = minimum;
for (int run = 1; run <= numberOfRuns; run++)
{
long time = results.Times[run];
sum += time;
minimum = Math.Min(minimum, time);
maximum = Math.Max(maximum, time);
}
long average = sum / (long)numberOfRuns;
double standardDeviation = 0.0;
if (numberOfRuns > 1)
{
double s = 0.0;
double a = (double)average;
double n = (double)numberOfRuns;
for (int run = 1; run <= numberOfRuns; run++)
{
double time = (double)results.Times[run];
double t = (time - a);
s += (t * t);
}
double variance = s / n;
if (a == 0.0)
{
standardDeviation = 0.0;
}
else
{
standardDeviation = 100.0 * (Math.Sqrt(variance) / a); // stddev as a percentage
standardDeviation = Math.Round(standardDeviation, 2, MidpointRounding.AwayFromZero);
}
}
// Record results and return.
results.Average = average;
results.Minimum = minimum;
results.Maximum = maximum;
results.StandardDeviation = standardDeviation;
results.Failures = failures;
return results;
}
public Results
(
Benchmark benchmark,
int numberOfRuns
)
{
Benchmark = benchmark;
Times = new long[numberOfRuns + 1]; // leave empty slot at index 0, not used.
Minimum = 0;
Maximum = 0;
Average = 0;
StandardDeviation = 0.0;
Failures = 0;
}
// Add a benchmark to the list of benchmarks read in from the .XML file.
public void AddBenchmark
(
string name,
string suiteName,
string tags,
string workingDirectory,
string exeName,
string exeArgs,
bool doRunInShell,
bool useSSE,
bool useAVX,
int expectedResults
)
{
BenchmarkSuite benchmarkSuite;
BenchmarkTagSet benchmarkTagSet;
Benchmark benchmark;
benchmark = new Benchmark(name, suiteName, tags,
workingDirectory, exeName, exeArgs, doRunInShell, useSSE, useAVX, expectedResults);
BenchmarkList.Add(benchmark);
if (!BenchmarkSuiteTable.TryGetValue(suiteName, out benchmarkSuite))
{
benchmarkSuite = new BenchmarkSuite(suiteName);
BenchmarkSuiteTable.Add(suiteName, benchmarkSuite);
}
benchmarkSuite.BenchmarkList.Add(benchmark);
string[] tagList = tags.Split(ListSeparatorCharSet, StringSplitOptions.RemoveEmptyEntries);
foreach (string tag in tagList)
{
if (!BenchmarkTagSetTable.TryGetValue(tag, out benchmarkTagSet))
{
benchmarkTagSet = new BenchmarkTagSet(tag);
BenchmarkTagSetTable.Add(tag, benchmarkTagSet);
}
benchmarkTagSet.BenchmarkList.Add(benchmark);
}
}
// Run benchmark - actually run benchmark the specified number of times and using the specified
// controls and return results.
public Results RunBenchmark(Benchmark benchmark)
{
bool doRun = Controls.DoRun;
int numberOfRuns = Controls.NumberOfRunsPerBenchmark;
bool doWarmUpRun = Controls.DoWarmUpRun;
string runner = Controls.Runner;
bool doDebugBenchmark = Controls.DoDebugBenchmark;
bool doVerbose = Controls.DoVerbose;
string complusVersion = Controls.ComplusVersion;
string benchmarksRootDirectory = Controls.BenchmarksRootDirectory;
string benchmarkDirectory = System.IO.Path.Combine(benchmarksRootDirectory, benchmark.WorkingDirectory);
bool doRunInShell = benchmark.DoRunInShell;
bool useSSE = benchmark.UseSSE;
bool useAVX = benchmark.UseAVX;
int expectedResults = benchmark.ExpectedResults;
int failureResults = ~expectedResults;
int actualResults;
int failures = 0;
Results results = new Results(benchmark, numberOfRuns);
if (!doRun)
{
return(results);
}
string workingDirectory = benchmarkDirectory;
string fileName = System.IO.Path.Combine(workingDirectory, benchmark.ExeName);
string args = benchmark.ExeArgs;
if (runner != "")
{
args = fileName + " " + args;
fileName = runner;
}
if (doDebugBenchmark)
{
args = "/debugexe " + fileName + " " + args;
fileName = "devenv.exe";
}
if (doRunInShell)
{
args = "/C " + fileName + " " + args;
fileName = "cmd.exe";
}
if (doVerbose)
{
Console.WriteLine("Running benchmark {0} ...", benchmark.Name);
Console.WriteLine("Invoking: {0} {1}", fileName, args);
}
for (int run = (doWarmUpRun) ? 0 : 1; run <= numberOfRuns; run++)
{
ProcessStartInfo startInfo = new ProcessStartInfo()
{
FileName = fileName,
Arguments = args,
WorkingDirectory = workingDirectory,
UseShellExecute = false
};
if (complusVersion != "")
{
startInfo.Environment["COMPlus_Version"] = complusVersion;
startInfo.Environment["COMPlus_DefaultVersion"] = complusVersion;
}
if (useSSE)
{
startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
startInfo.Environment["COMPlus_EnableAVX"] = "0";
}
if (useAVX)
{
startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
startInfo.Environment["COMPlus_EnableAVX"] = "1";
}
startInfo.Environment["COMPlus_gcConcurrent"] = "0";
startInfo.Environment["COMPlus_gcServer"] = "0";
startInfo.Environment["COMPlus_NoGuiOnAssert"] = "1";
startInfo.Environment["COMPlus_BreakOnUncaughtException"] = "0";
var clockTime = Stopwatch.StartNew();
int exitCode = 0;
try
{
using (var proc = Process.Start(startInfo))
{
proc.EnableRaisingEvents = true;
proc.WaitForExit();
exitCode = proc.ExitCode;
}
this.NumberOfBenchmarksRun++;
}
catch (Exception exception)
{
Console.WriteLine("Could not launch test {0} exception: {1}",
startInfo.FileName, exception);
exitCode = failureResults;
}
clockTime.Stop();
actualResults = exitCode;
long time = clockTime.ElapsedMilliseconds;
if (actualResults != expectedResults)
{
failures++;
time = 0;
}
results.Times[run] = time;
if (doVerbose)
{
Console.Write("Iteration benchmark {0} ", benchmark.Name);
if (actualResults == expectedResults)
{
Console.Write("elapsed time {0}ms", time);
}
else
{
Console.Write("FAILED(expected={0}, actual={1})", expectedResults, actualResults);
}
if (run == 0)
{
Console.Write(" (warmup)");
}
Console.WriteLine("");
}
}
// Calculate min, max, avg, and std devation.
long sum = 0;
long minimum = results.Times[1];
long maximum = minimum;
for (int run = 1; run <= numberOfRuns; run++)
{
long time = results.Times[run];
sum += time;
minimum = Math.Min(minimum, time);
maximum = Math.Max(maximum, time);
}
long average = sum / (long)numberOfRuns;
double standardDeviation = 0.0;
if (numberOfRuns > 1)
{
double s = 0.0;
double a = (double)average;
double n = (double)numberOfRuns;
for (int run = 1; run <= numberOfRuns; run++)
{
double time = (double)results.Times[run];
double t = (time - a);
s += (t * t);
}
double variance = s / n;
if (a == 0.0)
{
standardDeviation = 0.0;
}
else
{
standardDeviation = 100.0 * (Math.Sqrt(variance) / a); // stddev as a percentage
standardDeviation = Math.Round(standardDeviation, 2, MidpointRounding.AwayFromZero);
}
}
// Record results and return.
results.Average = average;
results.Minimum = minimum;
results.Maximum = maximum;
results.StandardDeviation = standardDeviation;
results.Failures = failures;
return(results);
}