/// <summary>
/// Runs the benchmark suite and returns the results.
/// </summary>
/// <returns>The results of the benchmark run</returns>
public RunResult Run()
{
ThrowIfCancellationRequested();
RunResult result = new RunResult();
result.Settings = m_run.Settings;
result.Environment = new RunEnvironment();
result.Environment.RunStartTime = DateTime.Now;
result.Environment.FrameworkDescription = RuntimeInformation.FrameworkDescription;
result.Environment.OperatingSystem = RuntimeInformation.OSDescription;
result.Environment.OSArchitecture = RuntimeInformation.OSArchitecture.ToString();
Logger.LogAlways($"Running benchmarks with server GC: {m_run.Settings.ServerGC}");
Logger.LogAlways($"Running benchmarks with concurrent GC: {m_run.Settings.ConcurrentGC}");
foreach (var version in m_run.CoreClrVersions)
{
// these should have been validated already before runnning
Debug.Assert(!string.IsNullOrEmpty(version.Path));
Debug.Assert(!string.IsNullOrEmpty(version.Name));
Debug.Assert(Directory.Exists(version.Path));
Debug.Assert(m_versionMap.ContainsKey(version));
PreparedCoreClrVersion preparedVersion = m_versionMap[version];
CoreclrVersionRunResult versionResult = RunVersion(preparedVersion);
result.PerVersionResults.Add(Tuple.Create(version, versionResult));
}
ThrowIfCancellationRequested();
return(result);
}
/// <summary>
/// Runs a single iteration of a benchmark. If no iteration number if specified,
/// the benchmark is run once.
/// </summary>
/// <param name="version">The version of CoreCLR to run the benchmark on</param>
/// <param name="bench">The benchmark to run</param>
/// <param name="termCondition">The termination condition for this benchmark</param>
/// <returns>The result of running the benchmark</returns>
private BenchmarkResult RunBenchmarkImplWithIterations(PreparedCoreClrVersion version, Benchmark bench, TerminationCondition termCondition)
{
ThrowIfCancellationRequested();
Logger.LogAlways($"Running iterations for benchmark {bench.Name}");
BenchmarkResult result = new BenchmarkResult();
result.Benchmark = bench;
int iterations = bench.Iterations ?? 1;
for (int i = 0; i < iterations; i++)
{
Logger.LogAlways($"Beginning iteration {i} for benchmark {bench.Name}");
// we'll create subdirectories for every iteration.
string folderName = Path.Combine(Directory.GetCurrentDirectory(), i.ToString());
Directory.CreateDirectory(folderName);
Directory.SetCurrentDirectory(folderName);
m_relativePath.Push(i.ToString());
try
{
IterationResult iterResult;
string traceName = bench.Name + ".etl";
m_traceCollector.StartTrace(bench.Name + ".etl", m_run.CollectionLevel);
try
{
// we've got everything set up, time to run.
iterResult = RunBenchmarkImpl(version, bench, termCondition);
}
finally
{
m_traceCollector.StopTrace();
}
var currentRelativePath = Path.Combine(m_relativePath.Reverse().ToArray());
// TODO(segilles, xplat) adding .zip on the end is done by PerfView, perfcollect
// probably doesn't do this.
iterResult.TracePathLocation = Path.Combine(currentRelativePath, traceName + ".zip");
// write out the result json file that the analysis engine is expecting
File.WriteAllText(
Path.Combine(Directory.GetCurrentDirectory(), Constants.ResultJsonName),
JsonConvert.SerializeObject(iterResult, Formatting.Indented));
result.Iterations.Add(iterResult);
}
finally
{
string upOneDir = Path.Combine(Directory.GetCurrentDirectory(), "..");
Directory.SetCurrentDirectory(upOneDir);
m_relativePath.Pop();
}
}
// write out the benchmark json
File.WriteAllText(
Path.Combine(Directory.GetCurrentDirectory(), Constants.BenchmarkJsonName),
JsonConvert.SerializeObject(bench, Formatting.Indented));
ThrowIfCancellationRequested();
return(result);
}
private static bool FixSingleCoreClrVersion(CoreClrVersion version, RunSettings settings, out PreparedCoreClrVersion preparedVersion)
{
// is this a fully-specified version (i.e. the user gave us a full directory)? if so, there's nothing
// to do here.
if (!version.IsPartial)
{
Logger.LogVerbose($"Version {version.Name} is complete and needs no additional processing");
preparedVersion = new PreparedCoreClrVersion(version);
return(true);
}
// otherwise, we're going to need to create a new directory and populate it.
// the idea here is to use the SharedBinaryFolder option given to us to populate
// the new folder we create and then copy the user provided files on top of it.
//
// we need settings.SharedBinaryFolder to be present for this, so we error here
// if it's not.
if (string.IsNullOrEmpty(settings.SharedBinaryFolder))
{
Logger.LogError("The SharedBinaryFolder settings key was required and not provided. See the documentation for more information.");
preparedVersion = null;
return(false);
}
if (!Directory.Exists(settings.SharedBinaryFolder))
{
Logger.LogError("The SharedBinaryFolder settings key referred to a folder that does not exist. See the documentation for more information.");
preparedVersion = null;
return(false);
}
Debug.Assert(Path.IsPathRooted(settings.SharedBinaryFolder));
var tempPath = Path.Combine(Path.GetTempPath(), "CoreGCBench.Runner", Guid.NewGuid().ToString());
Logger.LogVerbose($"Version {version.Name} is partial, using temp directory {tempPath}");
Directory.CreateDirectory(tempPath);
Logger.LogVerbose($"Copying files from {settings.SharedBinaryFolder} to {tempPath}");
CopyDirectory(version.Path, tempPath);
foreach (var file in version.Files)
{
Logger.LogVerbose($"Copying file {file} from {version.Path} to {tempPath}");
string filePath = Path.Combine(version.Path, file);
if (!File.Exists(filePath))
{
Logger.LogError($"File {file} for version {version.Name} does not exist at location {version.Path}");
preparedVersion = null;
return(false);
}
string targetPath = filePath.Replace(version.Path, tempPath);
File.Copy(filePath, targetPath);
}
Logger.LogVerbose($"Successfully created temporary version folder at {tempPath}");
preparedVersion = new PreparedCoreClrVersion(version, tempPath);
return(true);
}
/// <summary>
/// Runs the benchmark suite on a single version of CoreCLR and
/// returns the results.
/// </summary>
/// <param name="coreRootPath">The path to CORE_ROOT for the version
/// of CoreCLR being tested.</param>
/// <returns>The results of this run</returns>
private CoreclrVersionRunResult RunVersion(PreparedCoreClrVersion version)
{
ThrowIfCancellationRequested();
Logger.LogAlways($"Beginning run of version \"{version.Name}\"");
CoreclrVersionRunResult result = new CoreclrVersionRunResult();
Debug.Assert(Directory.GetCurrentDirectory() == m_options.OutputDirectory);
// TODO(segilles) error handling here. We should avoid propegating exceptions
// as best we can.
string folderName = Path.Combine(Directory.GetCurrentDirectory(), version.Name);
Directory.CreateDirectory(folderName);
Directory.SetCurrentDirectory(folderName);
m_relativePath.Push(version.Name);
try
{
foreach (var benchmark in m_run.Suite)
{
Debug.Assert(benchmark != null);
BenchmarkResult benchResult = RunBenchmark(version, benchmark);
result.BenchmarkResults.Add(benchResult);
}
// write out the version description
File.WriteAllText(
Path.Combine(folderName, Constants.VersionJsonName),
JsonConvert.SerializeObject(version));
ThrowIfCancellationRequested();
return(result);
}
finally
{
string upOneDir = Path.Combine(Directory.GetCurrentDirectory(), "..");
Directory.SetCurrentDirectory(upOneDir);
m_relativePath.Pop();
}
}
/// <summary>
/// Runs a single benchmark by spawning a process and monitoring it until
/// its exit.
/// </summary>
/// <param name="version">The coreclr version to test</param>
/// <param name="bench">The benchmark to run</param>
/// <param name="termCondition">The termination condition for this benchmark</param>
/// <returns>The result from running the benchmark</returns>
private IterationResult RunBenchmarkImpl(PreparedCoreClrVersion version, Benchmark bench, TerminationCondition termCondition)
{
ThrowIfCancellationRequested();
string coreRun = Path.Combine(version.Path, Utils.CoreRunName);
string arguments = bench.Arguments ?? "";
Debug.Assert(File.Exists(coreRun));
Debug.Assert(m_executableProbeMap.ContainsKey(bench));
string exePath = m_executableProbeMap[bench];
Process proc = new Process();
proc.StartInfo.FileName = coreRun;
proc.StartInfo.Arguments = exePath + " " + arguments;
proc.StartInfo.UseShellExecute = false;
proc.StartInfo.CreateNoWindow = false;
foreach (var pair in bench.EnvironmentVariables)
{
proc.StartInfo.Environment[pair.Key] = pair.Value;
}
proc.StartInfo.Environment[Constants.ServerGCVariable] = m_run.Settings.ServerGC ? "1" : "0";
proc.StartInfo.Environment[Constants.ConcurrentGCVariable] = m_run.Settings.ConcurrentGC ? "1" : "0";
// run the process!
RunProcess(termCondition, proc);
Debug.Assert(proc.HasExited);
IterationResult result = new IterationResult();
result.DurationMsec = (long)(proc.ExitTime - proc.StartTime).TotalMilliseconds;
result.ExitCode = proc.ExitCode;
result.Pid = proc.Id;
ThrowIfCancellationRequested();
return(result);
}
/// <summary>
/// Runs a single benchmark on a given version of CoreCLR and saves the results.
/// </summary>
/// <param name="version">The version of CoreCLR to run on</param>
/// <param name="bench">The benchmark to run</param>
/// <returns>The result from running the benchmark</returns>
private BenchmarkResult RunBenchmark(PreparedCoreClrVersion version, Benchmark bench)
{
ThrowIfCancellationRequested();
Logger.LogAlways($"Running benchmark {bench.Name}");
string folderName = Path.Combine(Directory.GetCurrentDirectory(), bench.Name);
Directory.CreateDirectory(folderName);
Directory.SetCurrentDirectory(folderName);
m_relativePath.Push(bench.Name);
try
{
using (TerminationCondition condition = ConstructTerminationCondition(bench))
{
ThrowIfCancellationRequested();
return(RunBenchmarkImplWithIterations(version, bench, condition));
}
}
finally
{
string upOneDir = Path.Combine(Directory.GetCurrentDirectory(), "..");
Directory.SetCurrentDirectory(upOneDir);
m_relativePath.Pop();
}
}
