private void ProcessResults(XUnitPerformanceSessionData xUnitSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData)
{
if (!File.Exists(Configuration.FileLogPath))
{
WriteWarningLine($"Results file '{Configuration.FileLogPath}' does not exist. Skipping processing of results.");
return;
}
var reader = new CSVMetricReader(Configuration.FileLogPath);
var fileNameWithoutExtension = $"{xUnitSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitSessionData.AssemblyFileName)}";
var assemblyModel = AssemblyModel.Create(xUnitSessionData.AssemblyFileName, reader, xUnitPerformanceMetricData);
var xmlFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.xml");
new AssemblyModelCollection {
assemblyModel
}.Serialize(xmlFileName);
xUnitSessionData.CollectOutputFilesCallback(xmlFileName);
var dt = assemblyModel.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
var mdFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.md");
MarkdownHelper.Write(mdFileName, mdTable);
xUnitSessionData.CollectOutputFilesCallback(mdFileName);
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
var csvFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.csv");
dt.WriteToCSV(csvFileName);
xUnitSessionData.CollectOutputFilesCallback(csvFileName);
BenchmarkEventSource.Log.Clear();
}
private void ProcessResults(string assemblyFileName, string sessionName, string outputDirectory, Action <string> collectOutputFilesCallback)
{
var reader = new CSVMetricReader(Configuration.FileLogPath);
var fileNameWithoutExtension = $"{sessionName}-{Path.GetFileNameWithoutExtension(assemblyFileName)}";
var statisticsFileName = $"{fileNameWithoutExtension}-Statistics";
var mdFileName = Path.Combine(outputDirectory, $"{statisticsFileName}.md");
var assemblyModel = AssemblyModel.Create(assemblyFileName, reader);
var xmlFileName = Path.Combine(outputDirectory, $"{fileNameWithoutExtension}.xml");
new AssemblyModelCollection {
assemblyModel
}.Serialize(xmlFileName);
WriteInfoLine($"Performance results saved to \"{xmlFileName}\"");
collectOutputFilesCallback(xmlFileName);
var dt = assemblyModel.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
MarkdownHelper.Write(mdFileName, mdTable);
WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
collectOutputFilesCallback(mdFileName);
Console.WriteLine(mdTable);
var csvFileName = Path.Combine(outputDirectory, $"{statisticsFileName}.csv");
dt.WriteToCSV(csvFileName);
WriteInfoLine($"Statistics written to \"{csvFileName}\"");
collectOutputFilesCallback(csvFileName);
BenchmarkEventSource.Log.Clear();
}
public void RunScenario(ProcessStartInfo processStartInfo, Action <ScenarioTestConfiguration> preIterationDel,
Action <ScenarioTestConfiguration> postIterationDel, Func <ScenarioTestConfiguration, ScenarioBenchmark> teardownDel)
{
int iterations = _scenarioTestConfiguration.Iterations;
int timeout = (int)(_scenarioTestConfiguration.TimeoutPerIteration.TotalMilliseconds);
for (int i = 0; i < iterations; i++)
{
preIterationDel(_scenarioTestConfiguration);
using (var p = new Process())
{
p.StartInfo = processStartInfo;
p.Start();
if (p.WaitForExit(timeout) == false)
{
p.Kill();
Console.WriteLine("Timeouted!");
return;
}
}
postIterationDel(_scenarioTestConfiguration);
}
ScenarioBenchmark scenarioBenchmark = teardownDel(_scenarioTestConfiguration);
if (scenarioBenchmark == null)
{
throw new InvalidOperationException("The Teardown Delegate should return a valid instance of ScenarioBenchmark.");
}
string scenarioNamespace = scenarioBenchmark.Namespace;
scenarioBenchmark.Serialize(Configuration.RunId + "-" + scenarioNamespace + ".xml");
var mdFileName = Configuration.RunId + "-" + scenarioNamespace + "-Statistics.md";
var csvFileName = Configuration.RunId + "-" + scenarioNamespace + "-Statistics.csv";
var dt = scenarioBenchmark.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
MarkdownHelper.Write(mdFileName, mdTable);
WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
Console.WriteLine(mdTable);
dt.WriteToCSV(csvFileName);
WriteInfoLine($"Statistics written to \"{csvFileName}\"");
}
/// <summary>
/// Saves Markdown and CSV results for executed scenarios
/// </summary>
/// <param name="scenarios">The scenarios to save results for</param>
/// <param name="fileNameWithoutExtension">The filename (without extension) to use to save results</param>
/// <param name="includeScenarioNameColumn">Indicates whether scenario name should be included in the tables as the first column</param>
public void WriteTableResults(IEnumerable <ScenarioBenchmark> scenarios, string fileNameWithoutExtension, bool includeScenarioNameColumn)
{
var dt = ScenarioBenchmark.GetEmptyTable(includeScenarioNameColumn ? null : scenarios.First().Name);
foreach (var scenario in scenarios)
{
scenario.AddRowsToTable(dt, scenario.GetStatistics(), includeScenarioNameColumn);
}
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
var csvFileName = $"{fileNameWithoutExtension}.csv";
dt.WriteToCSV(csvFileName);
LogFileSaved(csvFileName);
var mdFileName = $"{fileNameWithoutExtension}.md";
MarkdownHelper.Write(mdFileName, mdTable);
LogFileSaved(mdFileName);
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
}
/// <summary>
/// 1. In the specified assembly, get the ETW providers set as assembly attributes (PerformanceTestInfo)
/// 2. Check if the benchmark assembly request Precise Machine Counters(PMC) to be collected
/// 3. Enable Kernel providers if needed
/// 4. Get non-kernel ETW flags set and enable them
/// 5. Run the benchmarks
/// 6. Stop collecting ETW
/// 7. Merge ETL files.
/// </summary>
/// <param name="xUnitPerformanceSessionData"></param>
/// <param name="xUnitPerformanceMetricData"></param>
/// <param name="action"></param>
public static void Record(XUnitPerformanceSessionData xUnitPerformanceSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData, Action action)
{
const int bufferSizeMB = 256;
var name = $"{xUnitPerformanceSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitPerformanceSessionData.AssemblyFileName)}";
var etwOutputData = new ETWOutputData {
KernelFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.kernel.etl"), // without this parameter, EnableKernelProvider will fail
Name = name,
SessionName = $"Performance-Api-Session-{xUnitPerformanceSessionData.RunId}",
UserFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.etl"),
};
PrintProfilingInformation(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData);
var kernelProviderInfo = xUnitPerformanceMetricData.Providers.OfType <KernelProviderInfo>().FirstOrDefault();
var needKernelSession = NeedSeparateKernelSession(kernelProviderInfo);
if (needKernelSession && !CanEnableEnableKernelProvider)
{
const string message = "In order to capture kernel data the application is required to run as Administrator.";
WriteErrorLine(message);
throw new InvalidOperationException(message);
}
WriteDebugLine("> ETW capture start.");
// Prior to Windows 8 (NT 6.2), all kernel events needed the special kernel session.
using (var safeKernelSession = needKernelSession && CanEnableEnableKernelProvider ? MakeSafeTerminateTraceEventSession(KernelTraceEventParser.KernelSessionName, etwOutputData.KernelFileName) : null)
{
var kernelSession = safeKernelSession?.BaseDisposableObject;
if (kernelSession != null)
{
SetPreciseMachineCounters(xUnitPerformanceMetricData.Providers);
kernelSession.BufferSizeMB = bufferSizeMB;
var flags = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords;
var stackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords;
kernelSession.EnableKernelProvider(flags, stackCapture);
}
using (var safeUserEventSession = MakeSafeTerminateTraceEventSession(etwOutputData.SessionName, etwOutputData.UserFileName))
{
var userEventSession = safeUserEventSession.BaseDisposableObject;
userEventSession.BufferSizeMB = bufferSizeMB;
if (needKernelSession && CanEnableEnableKernelProvider)
{
userEventSession.EnableKernelProvider(KernelProvider.Default.Flags, KernelProvider.Default.StackCapture);
}
foreach (var provider in UserProvider.Defaults)
{
userEventSession.EnableProvider(provider.Guid, provider.Level, provider.Keywords);
}
foreach (var provider in xUnitPerformanceMetricData.Providers.OfType <UserProviderInfo>())
{
userEventSession.EnableProvider(provider.ProviderGuid, provider.Level, provider.Keywords);
}
action.Invoke();
}
}
WriteDebugLine("> ETW capture stop.");
// TODO: Decouple the code below.
// Collect ETW profile data.
// TODO: Skip collecting kernel data if it was not captured! (data will be zero, there is no point to report it or upload it)
WriteDebugLine("> ETW merge start.");
TraceEventSession.MergeInPlace(etwOutputData.UserFileName, Console.Out);
WriteDebugLine("> ETW merge stop.");
xUnitPerformanceSessionData.CollectOutputFilesCallback(etwOutputData.UserFileName);
var assemblyModel = GetAssemblyModel(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData.UserFileName, xUnitPerformanceSessionData.RunId, xUnitPerformanceMetricData.PerformanceTestMessages);
var xmlFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.xml");
new AssemblyModelCollection {
assemblyModel
}.Serialize(xmlFileName);
xUnitPerformanceSessionData.CollectOutputFilesCallback(xmlFileName);
var mdFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.md");
var dt = assemblyModel.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
MarkdownHelper.Write(mdFileName, mdTable);
xUnitPerformanceSessionData.CollectOutputFilesCallback(mdFileName);
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
var csvFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.csv");
dt.WriteToCSV(csvFileName);
xUnitPerformanceSessionData.CollectOutputFilesCallback(csvFileName);
}
/// <summary>
/// Executes the benchmark scenario specified by the parameter
/// containing the process start information.<br/>
/// The process component will wait, for the benchmark scenario to exit,
/// the time specified on the configuration argument.<br/>
/// If the benchmark scenario has not exited, then it will immediately
/// stop the associated process, and a TimeoutException will be thrown.
/// </summary>
/// <param name="processStartInfo">The ProcessStartInfo that contains the information that is used to start the benchmark scenario process.</param>
/// <param name="preIterationDelegate">The action that will be executed before every benchmark scenario execution.</param>
/// <param name="postIterationDelegate">The action that will be executed after every benchmark scenario execution.</param>
/// <param name="teardownDelegate">The action that will be executed after running all benchmark scenario iterations.</param>
/// <param name="scenarioConfiguration">ScenarioConfiguration object that defined the scenario execution.</param>
public void RunScenario(
ProcessStartInfo processStartInfo,
Action preIterationDelegate,
Action postIterationDelegate,
Func <ScenarioBenchmark> teardownDelegate,
ScenarioConfiguration scenarioConfiguration)
{
if (processStartInfo == null)
{
throw new ArgumentNullException($"{nameof(processStartInfo)} cannot be null.");
}
if (teardownDelegate == null)
{
throw new ArgumentNullException($"{nameof(teardownDelegate)} cannot be null.");
}
if (scenarioConfiguration == null)
{
throw new ArgumentNullException($"{nameof(scenarioConfiguration)} cannot be null.");
}
// Make a copy of the user input to avoid potential bugs due to changes behind the scenes.
var configuration = new ScenarioConfiguration(scenarioConfiguration);
for (int i = 0; i < configuration.Iterations; ++i)
{
preIterationDelegate?.Invoke();
// TODO: Start scenario profiling.
using (var process = new Process())
{
process.StartInfo = processStartInfo;
process.Start();
if (process.WaitForExit((int)(configuration.TimeoutPerIteration.TotalMilliseconds)) == false)
{
process.Kill();
throw new TimeoutException("Running benchmark scenario has timed out.");
}
// Check for the exit code.
if (!configuration.SuccessExitCodes.Contains(process.ExitCode))
{
throw new Exception($"'{processStartInfo.FileName}' exited with an invalid exit code: {process.ExitCode}");
}
}
// TODO: Stop scenario profiling.
postIterationDelegate?.Invoke();
}
ScenarioBenchmark scenarioBenchmark = teardownDelegate();
if (scenarioBenchmark == null)
{
throw new InvalidOperationException("The Teardown Delegate should return a valid instance of ScenarioBenchmark.");
}
scenarioBenchmark.Serialize(Configuration.RunId + "-" + scenarioBenchmark.Namespace + ".xml");
var dt = scenarioBenchmark.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
var mdFileName = $"{Configuration.RunId}-{scenarioBenchmark.Namespace}-Statistics.md";
MarkdownHelper.Write(mdFileName, mdTable);
WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
var csvFileName = $"{Configuration.RunId}-{scenarioBenchmark.Namespace}-Statistics.csv";
dt.WriteToCSV(csvFileName);
WriteInfoLine($"Statistics written to \"{csvFileName}\"");
}
/// <summary>
/// 1. In the specified assembly, get the ETW providers set as assembly attributes (PerformanceTestInfo)
/// 2. Check if the benchmark assembly request Precise Machine Counters(PMC) to be collected
/// 3. Enable Kernel providers if needed
/// 4. Get non-kernel ETW flags set and enable them
/// 5. Run the benchmarks
/// 6. Stop collecting ETW
/// 7. Merge ETL files.
/// </summary>
/// <param name="assemblyFileName"></param>
/// <param name="runId"></param>
/// <param name="outputDirectory"></param>
/// <param name="action"></param>
/// <param name="collectOutputFilesCallback">Callback used to collect a list of files generated.</param>
/// <returns></returns>
public static void Record(string assemblyFileName, string runId, string outputDirectory, Action action, Action <string> collectOutputFilesCallback)
{
if (TraceEventSession.IsElevated() != true)
{
const string errMessage = "In order to profile, application is required to run as Administrator.";
WriteErrorLine(errMessage);
throw new InvalidOperationException(errMessage);
}
const int bufferSizeMB = 256;
var sessionName = $"Performance-Api-Session-{runId}";
var name = $"{runId}-{Path.GetFileNameWithoutExtension(assemblyFileName)}";
var userFullFileName = Path.Combine(outputDirectory, $"{name}.etl");
var kernelFullFileName = Path.Combine(outputDirectory, $"{name}.kernel.etl"); // without this parameter, EnableKernelProvider will fail
PrintProfilingInformation(assemblyFileName, sessionName, userFullFileName);
(var providers, var performanceTestMessages) = XunitBenchmark.GetMetadata(assemblyFileName);
var kernelProviderInfo = providers.OfType <KernelProviderInfo>().FirstOrDefault();
var needKernelSession = NeedSeparateKernelSession(kernelProviderInfo);
using (var safeKernelSession = needKernelSession ? MakeSafeTerminateTraceEventSession(KernelTraceEventParser.KernelSessionName, kernelFullFileName) : null)
{
var kernelSession = safeKernelSession?.BaseDisposableObject;
if (kernelSession != null)
{
SetPreciseMachineCounters(providers);
kernelSession.BufferSizeMB = bufferSizeMB;
var flags = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords;
var stackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords;
kernelSession.EnableKernelProvider(flags, stackCapture);
}
using (var safeUserEventSession = MakeSafeTerminateTraceEventSession(sessionName, userFullFileName))
{
var userEventSession = safeUserEventSession.BaseDisposableObject;
userEventSession.BufferSizeMB = bufferSizeMB;
var flags = KernelTraceEventParser.Keywords.Process | KernelTraceEventParser.Keywords.ImageLoad | KernelTraceEventParser.Keywords.Thread;
var stackCapture = KernelTraceEventParser.Keywords.Profile | KernelTraceEventParser.Keywords.ContextSwitch;
userEventSession.EnableKernelProvider(flags, stackCapture);
foreach (var userProviderInfo in providers.OfType <UserProviderInfo>())
{
userEventSession.EnableProvider(userProviderInfo.ProviderGuid, userProviderInfo.Level, userProviderInfo.Keywords);
}
action.Invoke();
}
}
TraceEventSession.MergeInPlace(userFullFileName, Console.Out);
WriteInfoLine($"ETW Tracing Session saved to \"{userFullFileName}\"");
collectOutputFilesCallback(userFullFileName);
var assemblyModel = GetAssemblyModel(assemblyFileName, userFullFileName, runId, performanceTestMessages);
var xmlFileName = Path.Combine(outputDirectory, $"{name}.xml");
new AssemblyModelCollection {
assemblyModel
}.Serialize(xmlFileName);
WriteInfoLine($"Performance results saved to \"{xmlFileName}\"");
collectOutputFilesCallback(xmlFileName);
var mdFileName = Path.Combine(outputDirectory, $"{name}.md");
var dt = assemblyModel.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
MarkdownHelper.Write(mdFileName, mdTable);
WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
collectOutputFilesCallback(mdFileName);
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
var csvFileName = Path.Combine(outputDirectory, $"{name}.csv");
dt.WriteToCSV(csvFileName);
WriteInfoLine($"Statistics written to \"{csvFileName}\"");
collectOutputFilesCallback(csvFileName);
}