/// <summary>
/// Given a benchmark run, resolves each benchmark's test executable by probing the path given to us.
/// </summary>
/// <param name="run">The BenchmarkRun whose benchmarks need to be resolved</param>
/// <param name="probeMap">The probeMap calculated for every benchmark in the set</param>
/// <returns>True if all benchmarks were resolved successfully, false otherwise.</returns>
private static bool ProbeForExecutables(BenchmarkRun run, out IDictionary <Benchmark, string> probeMap)
{
Logger.LogVerbose("Beginning test executable probe");
Dictionary <Benchmark, string> map = new Dictionary <Benchmark, string>();
foreach (var bench in run.Suite)
{
if (Path.IsPathRooted(bench.ExecutablePath))
{
Logger.LogError($"Benchmark {bench.Name} has an absolute path for its executable - please change this to a path relative to the TestProbeRoot.");
probeMap = null;
return(false);
}
string absolutePath;
if (!ProbeForExecutable(run.Settings.TestProbeRoot, bench.ExecutablePath, out absolutePath))
{
Logger.LogError($"Failed to locate test executable for {bench.Name}, probing from directory {run.Settings.TestProbeRoot}!");
probeMap = null;
return(false);
}
Debug.Assert(Path.IsPathRooted(absolutePath));
map[bench] = absolutePath;
}
probeMap = map;
Logger.LogVerbose("Test executable probe complete");
return(true);
}
/// <summary>
/// Constructs a new runner that will run the given suite
/// with the given options.
/// </summary>
/// <param name="suite">The benchmark suite to run. Must be validated already.</param>
/// <param name="options">The options governing the benchmark run</param>
public Runner(BenchmarkRun suite,
Options options,
IDictionary <Benchmark, string> executableProbeMap,
IDictionary <CoreClrVersion, PreparedCoreClrVersion> versionMap)
{
Debug.Assert(suite != null);
Debug.Assert(options != null);
m_run = suite;
m_options = options;
m_traceCollector = TraceCollectorFactory.Create();
m_executableProbeMap = executableProbeMap;
m_versionMap = versionMap;
}
/// <summary>
/// "Fixes" a CoreClrVersion by turning a possibly-incomplete CoreClrVersion definition into a full, runnable version.
/// Instead of providing a full "core root" directory for eacy coreclr version, users can provide individual binaries
/// for each version and provide a base "shared binary folder" from which all of the other binaries will be derived from.
/// This step ensures that all versions are fixed before running.
/// </summary>
/// <param name="run">The Benchmark whose versions will be fixed</param>
/// <param name="fixedVersions">The map of all fixed versions, if the fixing was successful.</param>
/// <returns></returns>
private static bool FixCoreClrVersions(BenchmarkRun run, out IDictionary <CoreClrVersion, PreparedCoreClrVersion> fixedVersions)
{
fixedVersions = new Dictionary <CoreClrVersion, PreparedCoreClrVersion>();
foreach (var version in run.CoreClrVersions)
{
PreparedCoreClrVersion preparedVersion;
if (!FixSingleCoreClrVersion(version, run.Settings, out preparedVersion))
{
return(false);
}
fixedVersions[version] = preparedVersion;
}
return(true);
}
private static bool LoadConfigFile(Options opts, out BenchmarkRun run)
{
try
{
string fileText;
if (opts.ConfigJson != null)
{
fileText = opts.ConfigJson;
}
else
{
fileText = File.ReadAllText(opts.ConfigFile);
}
run = JsonConvert.DeserializeObject <BenchmarkRun>(fileText);
return(true);
}
catch (Exception exn)
{
// TODO(segilles) we can probably produce a better error message here.
Logger.LogError($"Failed to load configuration file: {exn.Message}");
run = null;
return(false);
}
}
/// <summary>
/// Validates a configuration given to us. Prints an error
/// message and returns false upon failure, returns true
/// upon success.
/// </summary>
/// <param name="run">The configuration to validate</param>
/// <returns>True if the validation was successful, false otherwise</returns>
private static bool ValidateConfig(BenchmarkRun run)
{
Logger.LogVerbose("Validating configuration");
foreach (var version in run.CoreClrVersions)
{
if (version.Name.Any(c => Path.GetInvalidPathChars().Contains(c)) ||
string.IsNullOrEmpty(version.Name))
{
Logger.LogError($"Version name \"{version.Name}\" has inapporpriate characters for a file path.");
return(false);
}
if (!Directory.Exists(version.Path))
{
Logger.LogError($"Version path {version.Path} does not exist.");
return(false);
}
string coreRun = Path.Combine(version.Path, Utils.CoreRunName);
if (!File.Exists(coreRun))
{
Logger.LogError($"Corerun not found on path {version.Path}.");
return(false);
}
}
if (run.CoreClrVersions.Count == 0)
{
Logger.LogError("Must provide at least one version of CoreCLR to test.");
return(false);
}
foreach (var benchmark in run.Suite)
{
if (benchmark.Name.Any(c => Path.GetInvalidPathChars().Contains(c)) || string.IsNullOrEmpty(benchmark.Name))
{
Logger.LogError($"Benchmark name \"{benchmark.Name}\" has inapporpriate characters for a file path.");
return(false);
}
}
if (!Path.IsPathRooted(run.Settings.TestProbeRoot))
{
Logger.LogError($"Probe path {run.Settings.TestProbeRoot} is not absolute!");
return(false);
}
if (run.Settings.SharedBinaryFolder != null)
{
if (!Path.IsPathRooted(run.Settings.SharedBinaryFolder))
{
Logger.LogError($"Shared binary path {run.Settings.SharedBinaryFolder} is not absolute!");
return(false);
}
string coreRun = Path.Combine(run.Settings.SharedBinaryFolder, Utils.CoreRunName);
if (!File.Exists(coreRun))
{
Logger.LogError($"Corerun not found on path {run.Settings.SharedBinaryFolder}.");
return(false);
}
}
Logger.LogVerbose("Validation successful");
return(true);
}
