BenchmarkRunResult MeasureIterations(TestRun run, BenchmarkConfiguration config, ITestOutputHelper output)
{
// The XunitPerformanceHarness is hardcoded to log to the console. It would be nice if the output was configurable somehow
// but in lieue of that we can redirect all console output with light hackery.
using (var redirector = new ConsoleRedirector(output))
{
// XunitPerformanceHarness expects to do the raw commandline parsing itself, but I really don't like that its default collection
// metric requires the use of ETW. Getting an admin console or admin VS instance isn't where most people start, its
// a small nuissance, and for these tests its often not needed/adds non-trivial overhead. I set the default to stopwatch if the
// perf:collect argument hasn't been specified, but that sadly requires that I pre-parse, interpret, and then re-format all the
// args to make that change :(
//
// In TestRun.ValidateMetricNames() I pre-check if ETW is going to be needed and give an error there rather than doing all the
// test setup (~1 minute?) and then giving the error after the user has probably wandered away. That also relies on some of this
// replicated command line parsing.
string[] args = new string[] { "--perf:collect", string.Join("+", run.MetricNames), "--perf:outputdir", run.OutputDir, "--perf:runid", run.BenchviewRunId };
using (var harness = new XunitPerformanceHarness(args))
{
ProcessStartInfo startInfo = new ProcessStartInfo(run.DotNetInstallation.DotNetExe, ExePath + " " + CommandLineArguments);
startInfo.WorkingDirectory = WorkingDirPath;
startInfo.RedirectStandardError = true;
startInfo.RedirectStandardOutput = true;
IEnumerable <KeyValuePair <string, string> > extraEnvVars = config.EnvironmentVariables.Concat(EnvironmentVariables).Append(new KeyValuePair <string, string>("DOTNET_MULTILEVEL_LOOKUP", "0"));
foreach (KeyValuePair <string, string> kv in extraEnvVars)
{
startInfo.Environment[kv.Key] = kv.Value;
}
output.WriteLine("XUnitPerfHarness doesn't log env vars it uses to run processes. To workaround, logging them here:");
output.WriteLine(string.Join(", ", extraEnvVars.Select(kv => kv.Key + "=" + kv.Value)));
BenchmarkRunResult result = new BenchmarkRunResult(this, config);
StringBuilder stderr = new StringBuilder();
StringBuilder stdout = new StringBuilder();
var scenarioConfiguration = new ScenarioTestConfiguration(TimeSpan.FromMinutes(60), startInfo)
{
//XUnitPerformanceHarness writes files to disk starting with {runid}-{ScenarioBenchmarkName}-{TestName}
TestName = (Name + "-" + config.Name).Replace(' ', '_'),
Scenario = new ScenarioBenchmark("EndToEnd"),
Iterations = run.Iterations,
PreIterationDelegate = scenario =>
{
stderr.Clear();
stdout.Clear();
scenario.Process.ErrorDataReceived += (object sender, DataReceivedEventArgs errorLine) =>
{
if (!string.IsNullOrEmpty(errorLine.Data))
{
stderr.AppendLine(errorLine.Data);
redirector.WriteLine("STDERROR: " + errorLine.Data);
}
};
scenario.Process.OutputDataReceived += (object sender, DataReceivedEventArgs outputLine) =>
{
stdout.AppendLine(outputLine.Data);
redirector.WriteLine(outputLine.Data);
};
},
PostIterationDelegate = scenarioResult =>
{
result.IterationResults.Add(RecordIterationMetrics(scenarioResult, stdout.ToString(), stderr.ToString(), redirector));
}
};
harness.RunScenario(scenarioConfiguration, sb => { BenchviewResultExporter.ConvertRunResult(sb, result); });
return(result);
}
}
}
static TestRun ConfigureTestRun(CommandLineOptions options)
{
TestRun run = new TestRun()
{
OutputDir = GetInitialWorkingDir(),
DotnetFrameworkVersion = options.MicrosoftNetCoreAppPackageVersion,
Iterations = 11
};
if (options.OutputDirectory != null)
{
run.OutputDir = options.OutputDirectory;
}
if (options.CoreCLRBinaryDir != null)
{
if (!Directory.Exists(options.CoreCLRBinaryDir))
{
throw new Exception("coreclr-bin-dir directory " + options.CoreCLRBinaryDir + " does not exist");
}
run.PrivateCoreCLRBinDir = options.CoreCLRBinaryDir;
}
else
{
string coreRootEnv = Environment.GetEnvironmentVariable("CORE_ROOT");
if (coreRootEnv != null)
{
if (!Directory.Exists(coreRootEnv))
{
throw new Exception("CORE_ROOT directory " + coreRootEnv + " does not exist");
}
run.PrivateCoreCLRBinDir = coreRootEnv;
}
else
{
//maybe we've got private coreclr binaries in our current directory? Use those if so.
string currentDirectory = Directory.GetCurrentDirectory();
if (File.Exists(Path.Combine(currentDirectory, "System.Private.CoreLib.dll")))
{
run.PrivateCoreCLRBinDir = currentDirectory;
}
else
{
// don't use private CoreCLR binaries
}
}
}
if (options.DotnetFrameworkVersion != null)
{
run.DotnetFrameworkVersion = options.DotnetFrameworkVersion;
}
if (options.DotnetSdkVersion != null)
{
run.DotnetSdkVersion = options.DotnetSdkVersion;
}
else
{
run.DotnetSdkVersion = DotNetSetup.GetCompatibleDefaultSDKVersionForRuntimeVersion(run.DotnetFrameworkVersion);
}
if (options.TargetArchitecture != null)
{
if (options.TargetArchitecture.Equals("x64", StringComparison.OrdinalIgnoreCase))
{
run.Architecture = Architecture.X64;
}
else if (options.TargetArchitecture.Equals("x86", StringComparison.OrdinalIgnoreCase))
{
run.Architecture = Architecture.X86;
}
else
{
throw new Exception("Unrecognized architecture " + options.TargetArchitecture);
}
}
else
{
run.Architecture = RuntimeInformation.ProcessArchitecture;
}
if (options.Iterations > 0)
{
run.Iterations = (int)options.Iterations;
}
run.UseExistingSetup = options.UseExistingSetup;
run.BenchviewRunId = options.RunId ?? "Unofficial";
run.MetricNames.AddRange(options.MetricNames);
run.Benchmarks.AddRange(GetBenchmarkSelection(options));
run.Configurations.AddRange(GetBenchmarkConfigurations(options));
return(run);
}
