/// <summary>
/// Creates a new <see cref="ScenarioTest"/> object
/// </summary>
/// <param name="configuration">The scenario configuration</param>
public ScenarioTest(ScenarioTestConfiguration configuration)
{
_disposedValue = false;
Process = new Process {
StartInfo = configuration.StartInfo,
};
}
private static ScenarioExecutionResult Run(ScenarioTestConfiguration configuration, ScenarioTest scenarioTest)
{
var hasStarted = scenarioTest.Process.Start();
var startTime = DateTime.UtcNow;
if (!hasStarted)
{
throw new Exception($"Failed to start {scenarioTest.Process.ProcessName}");
}
if (scenarioTest.Process.StartInfo.RedirectStandardError)
{
scenarioTest.Process.BeginErrorReadLine();
}
if (scenarioTest.Process.StartInfo.RedirectStandardInput)
{
throw new NotSupportedException($"RedirectStandardInput is not currently supported.");
}
if (scenarioTest.Process.StartInfo.RedirectStandardOutput)
{
scenarioTest.Process.BeginOutputReadLine();
}
var hasExited = scenarioTest.Process.WaitForExit(
(int)(configuration.TimeoutPerIteration.TotalMilliseconds));
var exitTime = DateTime.UtcNow;
if (!hasExited)
{
// TODO: scenarioOutput.Process.Kill[All|Tree]();
scenarioTest.Process.Kill();
throw new TimeoutException("Running benchmark scenario has timed out.");
}
// Check for the exit code.
if (!configuration.SuccessExitCodes.Contains(scenarioTest.Process.ExitCode))
{
throw new Exception($"'{scenarioTest.Process.StartInfo.FileName}' exited with an invalid exit code: {scenarioTest.Process.ExitCode}");
}
return(new ScenarioExecutionResult(scenarioTest.Process, startTime, exitTime, configuration));
}
public XunitPerformanceHarness(string[] args)
{
_args = args;
_disposed = false;
_outputFiles = new List <string>();
_scenarioTestConfiguration = new ScenarioTestConfiguration();
var options = XunitPerformanceHarnessOptions.Parse(args);
_scenarioTestConfiguration.TemporaryDirectory = options.TemporaryDirectory;
// Set the run id.
_outputDirectory = options.OutputDirectory;
_typeNames = new List <string>(options.TypeNames);
_runner = (assemblyPath) =>
{
XunitRunner.Run(assemblyPath, _typeNames);
};
Configuration.RunId = options.RunId;
// Set the file log path.
// TODO: Conditionally set this based on whether we want a csv file written.
Configuration.FileLogPath = Configuration.RunId + ".csv";
}
/// <summary>
/// Executes the benchmark scenario specified by the parameter
/// containing the process start information.<br/>
/// The process component will wait, for the benchmark scenario to exit,
/// the time specified on the configuration argument.<br/>
/// If the benchmark scenario has not exited, then it will immediately
/// stop the associated process, and a TimeoutException will be thrown.
/// </summary>
/// <param name="configuration">ScenarioConfiguration object that defined the scenario execution.</param>
/// <param name="teardownDelegate">The action that will be executed after running all benchmark scenario iterations.</param>
public void RunScenario(ScenarioTestConfiguration configuration, Action <ScenarioBenchmark> teardownDelegate)
{
if (configuration == null)
{
throw new ArgumentNullException(nameof(configuration));
}
if (teardownDelegate == null)
{
throw new ArgumentNullException(nameof(teardownDelegate));
}
string testName;
if (configuration.TestName == null)
{
testName = Path.GetFileNameWithoutExtension(configuration.StartInfo.FileName);
if (configuration.Scenario == null)
{
configuration.Scenario = new ScenarioBenchmark(testName);
}
}
else
{
if (configuration.Scenario == null)
{
testName = configuration.TestName;
configuration.Scenario = new ScenarioBenchmark(testName);
}
else
{
testName = $"{configuration.Scenario.Name}-{configuration.TestName}";
}
}
var scenarioFileName = $"{Configuration.RunId}-{testName}";
var fileNameWithoutExtension = Path.Combine(OutputDirectory, $"{scenarioFileName}");
for (int i = 0; i < configuration.Iterations; ++i)
{
using (var scenarioTest = new ScenarioTest(configuration))
{
ScenarioExecutionResult scenarioExecutionResult;
configuration.PreIterationDelegate?.Invoke(scenarioTest);
WriteInfoLine($"Iteration ({i})");
WriteInfoLine($" Working Directory: \"{scenarioTest.Process.StartInfo.WorkingDirectory}\"");
WriteInfoLine($" Command: \"{scenarioTest.Process.StartInfo.FileName}\" {scenarioTest.Process.StartInfo.Arguments}");
if (IsWindowsPlatform && _requireEtw)
{
var sessionName = $"Performance-Api-Session-{Configuration.RunId}";
string tracesFolder = Path.Combine(OutputDirectory, $"{fileNameWithoutExtension}-traces");
if (!Directory.Exists(tracesFolder))
{
Directory.CreateDirectory(tracesFolder);
}
var etlFileName = Path.Combine(tracesFolder, $"{scenarioFileName}({i}).etl");
var userSpecifiedMetrics = _metricCollectionFactory.GetMetrics();
var kernelProviders = userSpecifiedMetrics
.SelectMany(pmi => pmi.ProviderInfo)
.OfType <KernelProviderInfo>()
.Select(kernelProviderInfo => new KernelProvider {
Flags = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords,
StackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords
});
var profileSourceInfos = userSpecifiedMetrics
.SelectMany(pmi => pmi.ProviderInfo)
.OfType <CpuCounterInfo>()
.Where(cpuCounterInfo => Helper.AvailablePreciseMachineCounters.Keys.Contains(cpuCounterInfo.CounterName))
.Select(cpuCounterInfo => {
var profileSourceInfo = Helper.AvailablePreciseMachineCounters[cpuCounterInfo.CounterName];
return(new ProfileSourceInfo {
ID = profileSourceInfo.ID,
Interval = cpuCounterInfo.Interval,
MaxInterval = profileSourceInfo.MaxInterval,
MinInterval = profileSourceInfo.MinInterval,
Name = profileSourceInfo.Name,
});
});
Helper.SetPreciseMachineCounters(profileSourceInfos.ToList());
var listener = new Listener <ScenarioExecutionResult>(
new SessionData(sessionName, etlFileName)
{
BufferSizeMB = 256
},
UserProvider.Defaults,
kernelProviders.ToList());
scenarioExecutionResult = listener.Record(() => { return(Run(configuration, scenarioTest)); });
scenarioExecutionResult.EventLogFileName = etlFileName;
scenarioExecutionResult.PerformanceMonitorCounters = userSpecifiedMetrics
.Where(m => Helper.AvailablePreciseMachineCounters.Keys.Contains(m.Id))
.Select(m => {
var psi = Helper.AvailablePreciseMachineCounters[m.Id];
return(new PerformanceMonitorCounter(m.DisplayName, psi.Name, m.Unit, psi.ID));
})
.ToHashSet();
LogFileSaved(etlFileName);
}
else
{
scenarioExecutionResult = Run(configuration, scenarioTest);
}
configuration.PostIterationDelegate?.Invoke(scenarioExecutionResult);
}
}
teardownDelegate(configuration.Scenario);
if (configuration.SaveResults)
{
WriteResults(configuration.Scenario, fileNameWithoutExtension);
}
}
/// <summary>
/// Initializes a new instance of the ScenarioExecutionResult class.
/// </summary>
/// <param name="process">Scenario benchmark process that was run.</param>
/// <param name="startTime">The time that the associated process was started.</param>
/// <param name="exitTime">The time that the associated process exited.</param>
/// <param name="configuration">Configuration for the scenario that was run.</param>
internal ScenarioExecutionResult(System.Diagnostics.Process process, DateTime startTime, DateTime exitTime, ScenarioTestConfiguration configuration)
{
ProcessExitInfo = new ProcessExitInfo(process, startTime, exitTime);
Configuration = configuration;
}