/// <summary>
/// Save results from an executed scenario in XML format
/// </summary>
/// <param name="scenario">The scenario to save results for</param>
/// <param name="fileNameWithoutExtension">The filename (without extension) to use to save results</param>
public void WriteXmlResults(ScenarioBenchmark scenario, string fileNameWithoutExtension)
{
var xmlFileName = $"{fileNameWithoutExtension}.xml";
scenario.Serialize(xmlFileName);
LogFileSaved(xmlFileName);
}
public void RunScenario(ProcessStartInfo processStartInfo, Action <ScenarioTestConfiguration> preIterationDel,
Action <ScenarioTestConfiguration> postIterationDel, Func <ScenarioTestConfiguration, ScenarioBenchmark> teardownDel)
{
int iterations = _scenarioTestConfiguration.Iterations;
int timeout = (int)(_scenarioTestConfiguration.TimeoutPerIteration.TotalMilliseconds);
for (int i = 0; i < iterations; i++)
{
preIterationDel(_scenarioTestConfiguration);
using (var p = new Process())
{
p.StartInfo = processStartInfo;
p.Start();
if (p.WaitForExit(timeout) == false)
{
p.Kill();
Console.WriteLine("Timeouted!");
return;
}
}
postIterationDel(_scenarioTestConfiguration);
}
ScenarioBenchmark scenarioBenchmark = teardownDel(_scenarioTestConfiguration);
if (scenarioBenchmark == null)
{
throw new InvalidOperationException("The Teardown Delegate should return a valid instance of ScenarioBenchmark.");
}
string scenarioNamespace = scenarioBenchmark.Namespace;
scenarioBenchmark.Serialize(Configuration.RunId + "-" + scenarioNamespace + ".xml");
var mdFileName = Configuration.RunId + "-" + scenarioNamespace + "-Statistics.md";
var csvFileName = Configuration.RunId + "-" + scenarioNamespace + "-Statistics.csv";
var dt = scenarioBenchmark.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
MarkdownHelper.Write(mdFileName, mdTable);
WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
Console.WriteLine(mdTable);
dt.WriteToCSV(csvFileName);
WriteInfoLine($"Statistics written to \"{csvFileName}\"");
}
/// <summary>
/// Saves Markdown and CSV results for executed scenarios
/// </summary>
/// <param name="scenarios">The scenarios to save results for</param>
/// <param name="fileNameWithoutExtension">The filename (without extension) to use to save results</param>
/// <param name="includeScenarioNameColumn">Indicates whether scenario name should be included in the tables as the first column</param>
public void WriteTableResults(IEnumerable <ScenarioBenchmark> scenarios, string fileNameWithoutExtension, bool includeScenarioNameColumn)
{
var dt = ScenarioBenchmark.GetEmptyTable(includeScenarioNameColumn ? null : scenarios.First().Name);
foreach (var scenario in scenarios)
{
scenario.AddRowsToTable(dt, scenario.GetStatistics(), includeScenarioNameColumn);
}
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
var csvFileName = $"{fileNameWithoutExtension}.csv";
dt.WriteToCSV(csvFileName);
LogFileSaved(csvFileName);
var mdFileName = $"{fileNameWithoutExtension}.md";
MarkdownHelper.Write(mdFileName, mdTable);
LogFileSaved(mdFileName);
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
}
/// <summary>
/// Save results from an executed scenario
/// </summary>
/// <param name="scenario">The scenario to save results for</param>
/// <param name="fileNameWithoutExtension">The filename (without extension) to use to save results</param>
/// <remarks>This will save an XML, a Markdown, and a CSV file with the results.</remarks>
public void WriteResults(ScenarioBenchmark scenario, string fileNameWithoutExtension)
{
WriteXmlResults(scenario, fileNameWithoutExtension);
WriteTableResults(new[] { scenario }, fileNameWithoutExtension, false);
}
/// <summary>
/// Executes the benchmark scenario specified by the parameter
/// containing the process start information.<br/>
/// The process component will wait, for the benchmark scenario to exit,
/// the time specified on the configuration argument.<br/>
/// If the benchmark scenario has not exited, then it will immediately
/// stop the associated process, and a TimeoutException will be thrown.
/// </summary>
/// <param name="processStartInfo">The ProcessStartInfo that contains the information that is used to start the benchmark scenario process.</param>
/// <param name="preIterationDelegate">The action that will be executed before every benchmark scenario execution.</param>
/// <param name="postIterationDelegate">The action that will be executed after every benchmark scenario execution.</param>
/// <param name="teardownDelegate">The action that will be executed after running all benchmark scenario iterations.</param>
/// <param name="scenarioConfiguration">ScenarioConfiguration object that defined the scenario execution.</param>
public void RunScenario(
ProcessStartInfo processStartInfo,
Action preIterationDelegate,
Action postIterationDelegate,
Func <ScenarioBenchmark> teardownDelegate,
ScenarioConfiguration scenarioConfiguration)
{
if (processStartInfo == null)
{
throw new ArgumentNullException($"{nameof(processStartInfo)} cannot be null.");
}
if (teardownDelegate == null)
{
throw new ArgumentNullException($"{nameof(teardownDelegate)} cannot be null.");
}
if (scenarioConfiguration == null)
{
throw new ArgumentNullException($"{nameof(scenarioConfiguration)} cannot be null.");
}
// Make a copy of the user input to avoid potential bugs due to changes behind the scenes.
var configuration = new ScenarioConfiguration(scenarioConfiguration);
for (int i = 0; i < configuration.Iterations; ++i)
{
preIterationDelegate?.Invoke();
// TODO: Start scenario profiling.
using (var process = new Process())
{
process.StartInfo = processStartInfo;
process.Start();
if (process.WaitForExit((int)(configuration.TimeoutPerIteration.TotalMilliseconds)) == false)
{
process.Kill();
throw new TimeoutException("Running benchmark scenario has timed out.");
}
// Check for the exit code.
if (!configuration.SuccessExitCodes.Contains(process.ExitCode))
{
throw new Exception($"'{processStartInfo.FileName}' exited with an invalid exit code: {process.ExitCode}");
}
}
// TODO: Stop scenario profiling.
postIterationDelegate?.Invoke();
}
ScenarioBenchmark scenarioBenchmark = teardownDelegate();
if (scenarioBenchmark == null)
{
throw new InvalidOperationException("The Teardown Delegate should return a valid instance of ScenarioBenchmark.");
}
scenarioBenchmark.Serialize(Configuration.RunId + "-" + scenarioBenchmark.Namespace + ".xml");
var dt = scenarioBenchmark.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
var mdFileName = $"{Configuration.RunId}-{scenarioBenchmark.Namespace}-Statistics.md";
MarkdownHelper.Write(mdFileName, mdTable);
WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
var csvFileName = $"{Configuration.RunId}-{scenarioBenchmark.Namespace}-Statistics.csv";
dt.WriteToCSV(csvFileName);
WriteInfoLine($"Statistics written to \"{csvFileName}\"");
}