// Run benchmark - actually run benchmark the specified number of times and using the specified
// controls and return results.
public Results RunBenchmark(Benchmark benchmark)
{
bool doRun = Controls.DoRun;
int numberOfRuns = Controls.NumberOfRunsPerBenchmark;
bool doWarmUpRun = Controls.DoWarmUpRun;
string runner = Controls.Runner;
bool doDebugBenchmark = Controls.DoDebugBenchmark;
bool doVerbose = Controls.DoVerbose;
string complusVersion = Controls.ComplusVersion;
string benchmarksRootDirectory = Controls.BenchmarksRootDirectory;
string benchmarkDirectory = System.IO.Path.Combine(benchmarksRootDirectory, benchmark.WorkingDirectory);
bool doRunInShell = benchmark.DoRunInShell;
bool useSSE = benchmark.UseSSE;
bool useAVX = benchmark.UseAVX;
int expectedResults = benchmark.ExpectedResults;
int failureResults = ~expectedResults;
int actualResults;
int failures = 0;
Results results = new Results(benchmark, numberOfRuns);
if (!doRun)
{
return results;
}
string workingDirectory = benchmarkDirectory;
string fileName = System.IO.Path.Combine(workingDirectory, benchmark.ExeName);
string args = benchmark.ExeArgs;
if (runner != "")
{
args = fileName + " " + args;
fileName = runner;
}
if (doDebugBenchmark)
{
args = "/debugexe " + fileName + " " + args;
fileName = "devenv.exe";
}
if (doRunInShell)
{
args = "/C " + fileName + " " + args;
fileName = "cmd.exe";
}
if (doVerbose)
{
Console.WriteLine("Running benchmark {0} ...", benchmark.Name);
Console.WriteLine("Invoking: {0} {1}", fileName, args);
}
for (int run = (doWarmUpRun) ? 0 : 1; run <= numberOfRuns; run++)
{
ProcessStartInfo startInfo = new ProcessStartInfo()
{
FileName = fileName,
Arguments = args,
WorkingDirectory = workingDirectory,
UseShellExecute = false
};
if (complusVersion != "")
{
startInfo.Environment["COMPlus_Version"] = complusVersion;
startInfo.Environment["COMPlus_DefaultVersion"] = complusVersion;
}
if (useSSE)
{
startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
startInfo.Environment["COMPlus_EnableAVX"] = "0";
}
if (useAVX)
{
startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
startInfo.Environment["COMPlus_EnableAVX"] = "1";
}
startInfo.Environment["COMPlus_gcConcurrent"] = "0";
startInfo.Environment["COMPlus_gcServer"] = "0";
startInfo.Environment["COMPlus_NoGuiOnAssert"] = "1";
startInfo.Environment["COMPlus_BreakOnUncaughtException"] = "0";
var clockTime = Stopwatch.StartNew();
int exitCode = 0;
try
{
using (var proc = Process.Start(startInfo))
{
proc.EnableRaisingEvents = true;
proc.WaitForExit();
exitCode = proc.ExitCode;
}
this.NumberOfBenchmarksRun++;
}
catch (Exception exception)
{
Console.WriteLine("Could not launch test {0} exception: {1}",
startInfo.FileName, exception);
exitCode = failureResults;
}
clockTime.Stop();
actualResults = exitCode;
long time = clockTime.ElapsedMilliseconds;
if (actualResults != expectedResults)
{
failures++;
time = 0;
}
results.Times[run] = time;
if (doVerbose)
{
Console.Write("Iteration benchmark {0} ", benchmark.Name);
if (actualResults == expectedResults)
{
Console.Write("elapsed time {0}ms", time);
}
else
{
Console.Write("FAILED(expected={0}, actual={1})", expectedResults, actualResults);
}
if (run == 0)
{
Console.Write(" (warmup)");
}
Console.WriteLine("");
}
}
// Calculate min, max, avg, and std devation.
long sum = 0;
long minimum = results.Times[1];
long maximum = minimum;
for (int run = 1; run <= numberOfRuns; run++)
{
long time = results.Times[run];
sum += time;
minimum = Math.Min(minimum, time);
maximum = Math.Max(maximum, time);
}
long average = sum / (long)numberOfRuns;
double standardDeviation = 0.0;
if (numberOfRuns > 1)
{
double s = 0.0;
double a = (double)average;
double n = (double)numberOfRuns;
for (int run = 1; run <= numberOfRuns; run++)
{
double time = (double)results.Times[run];
double t = (time - a);
s += (t * t);
}
double variance = s / n;
if (a == 0.0)
{
standardDeviation = 0.0;
}
else
{
standardDeviation = 100.0 * (Math.Sqrt(variance) / a); // stddev as a percentage
standardDeviation = Math.Round(standardDeviation, 2, MidpointRounding.AwayFromZero);
}
}
// Record results and return.
results.Average = average;
results.Minimum = minimum;
results.Maximum = maximum;
results.StandardDeviation = standardDeviation;
results.Failures = failures;
return results;
}
// Run benchmark - actually run benchmark the specified number of times and using the specified
// controls and return results.
public Results RunBenchmark(Benchmark benchmark)
{
bool doRun = Controls.DoRun;
int numberOfRuns = Controls.NumberOfRunsPerBenchmark;
bool doWarmUpRun = Controls.DoWarmUpRun;
string runner = Controls.Runner;
bool doDebugBenchmark = Controls.DoDebugBenchmark;
bool doVerbose = Controls.DoVerbose;
string complusVersion = Controls.ComplusVersion;
string benchmarksRootDirectory = Controls.BenchmarksRootDirectory;
string benchmarkDirectory = System.IO.Path.Combine(benchmarksRootDirectory, benchmark.WorkingDirectory);
bool doRunInShell = benchmark.DoRunInShell;
bool useSSE = benchmark.UseSSE;
bool useAVX = benchmark.UseAVX;
int expectedResults = benchmark.ExpectedResults;
int failureResults = ~expectedResults;
int actualResults;
int failures = 0;
Results results = new Results(benchmark, numberOfRuns);
if (!doRun)
{
return(results);
}
string workingDirectory = benchmarkDirectory;
string fileName = System.IO.Path.Combine(workingDirectory, benchmark.ExeName);
string args = benchmark.ExeArgs;
if (runner != "")
{
args = fileName + " " + args;
fileName = runner;
}
if (doDebugBenchmark)
{
args = "/debugexe " + fileName + " " + args;
fileName = "devenv.exe";
}
if (doRunInShell)
{
args = "/C " + fileName + " " + args;
fileName = "cmd.exe";
}
if (doVerbose)
{
Console.WriteLine("Running benchmark {0} ...", benchmark.Name);
Console.WriteLine("Invoking: {0} {1}", fileName, args);
}
for (int run = (doWarmUpRun) ? 0 : 1; run <= numberOfRuns; run++)
{
ProcessStartInfo startInfo = new ProcessStartInfo()
{
FileName = fileName,
Arguments = args,
WorkingDirectory = workingDirectory,
UseShellExecute = false
};
if (complusVersion != "")
{
startInfo.Environment["COMPlus_Version"] = complusVersion;
startInfo.Environment["COMPlus_DefaultVersion"] = complusVersion;
}
if (useSSE)
{
startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
startInfo.Environment["COMPlus_EnableAVX"] = "0";
}
if (useAVX)
{
startInfo.Environment["COMPlus_FeatureSIMD"] = "1";
startInfo.Environment["COMPlus_EnableAVX"] = "1";
}
startInfo.Environment["COMPlus_gcConcurrent"] = "0";
startInfo.Environment["COMPlus_gcServer"] = "0";
startInfo.Environment["COMPlus_NoGuiOnAssert"] = "1";
startInfo.Environment["COMPlus_BreakOnUncaughtException"] = "0";
var clockTime = Stopwatch.StartNew();
int exitCode = 0;
try
{
using (var proc = Process.Start(startInfo))
{
proc.EnableRaisingEvents = true;
proc.WaitForExit();
exitCode = proc.ExitCode;
}
this.NumberOfBenchmarksRun++;
}
catch (Exception exception)
{
Console.WriteLine("Could not launch test {0} exception: {1}",
startInfo.FileName, exception);
exitCode = failureResults;
}
clockTime.Stop();
actualResults = exitCode;
long time = clockTime.ElapsedMilliseconds;
if (actualResults != expectedResults)
{
failures++;
time = 0;
}
results.Times[run] = time;
if (doVerbose)
{
Console.Write("Iteration benchmark {0} ", benchmark.Name);
if (actualResults == expectedResults)
{
Console.Write("elapsed time {0}ms", time);
}
else
{
Console.Write("FAILED(expected={0}, actual={1})", expectedResults, actualResults);
}
if (run == 0)
{
Console.Write(" (warmup)");
}
Console.WriteLine("");
}
}
// Calculate min, max, avg, and std devation.
long sum = 0;
long minimum = results.Times[1];
long maximum = minimum;
for (int run = 1; run <= numberOfRuns; run++)
{
long time = results.Times[run];
sum += time;
minimum = Math.Min(minimum, time);
maximum = Math.Max(maximum, time);
}
long average = sum / (long)numberOfRuns;
double standardDeviation = 0.0;
if (numberOfRuns > 1)
{
double s = 0.0;
double a = (double)average;
double n = (double)numberOfRuns;
for (int run = 1; run <= numberOfRuns; run++)
{
double time = (double)results.Times[run];
double t = (time - a);
s += (t * t);
}
double variance = s / n;
if (a == 0.0)
{
standardDeviation = 0.0;
}
else
{
standardDeviation = 100.0 * (Math.Sqrt(variance) / a); // stddev as a percentage
standardDeviation = Math.Round(standardDeviation, 2, MidpointRounding.AwayFromZero);
}
}
// Record results and return.
results.Average = average;
results.Minimum = minimum;
results.Maximum = maximum;
results.StandardDeviation = standardDeviation;
results.Failures = failures;
return(results);
}