private void ProcessResults(XUnitPerformanceSessionData xUnitSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData)
{
if (!File.Exists(Configuration.FileLogPath))
{
WriteWarningLine($"Results file '{Configuration.FileLogPath}' does not exist. Skipping processing of results.");
return;
}
var reader = new CSVMetricReader(Configuration.FileLogPath);
var fileNameWithoutExtension = $"{xUnitSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitSessionData.AssemblyFileName)}";
var assemblyModel = AssemblyModel.Create(xUnitSessionData.AssemblyFileName, reader, xUnitPerformanceMetricData);
var xmlFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.xml");
new AssemblyModelCollection {
assemblyModel
}.Serialize(xmlFileName);
xUnitSessionData.CollectOutputFilesCallback(xmlFileName);
var dt = assemblyModel.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
var mdFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.md");
MarkdownHelper.Write(mdFileName, mdTable);
xUnitSessionData.CollectOutputFilesCallback(mdFileName);
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
var csvFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.csv");
dt.WriteToCSV(csvFileName);
xUnitSessionData.CollectOutputFilesCallback(csvFileName);
BenchmarkEventSource.Log.Clear();
}
/// <summary>
/// Run the xUnit tests tagged with the [<see cref="BenchmarkAttribute"/>] attribute.
/// </summary>
/// <param name="assemblyFileName">Path to the assembly that contains the xUnit performance tests.</param>
public void RunBenchmarks(string assemblyFileName)
{
if (string.IsNullOrEmpty(assemblyFileName))
{
throw new ArgumentNullException(nameof(assemblyFileName));
}
if (!File.Exists(assemblyFileName))
{
throw new FileNotFoundException(assemblyFileName);
}
void xUnitAction(string assemblyPath)
{
var errCode = XunitRunner.Run(assemblyPath, _typeNames);
if (errCode != 0)
{
throw new Exception($"{errCode} benchmark(s) failed to execute.");
}
}
var xUnitPerformanceSessionData = new XUnitPerformanceSessionData
{
AssemblyFileName = assemblyFileName,
CollectOutputFilesCallback = LogFileSaved,
OutputDirectory = OutputDirectory,
RunId = Configuration.RunId
};
var xUnitPerformanceMetricData = XunitBenchmark.GetMetadata(
assemblyFileName,
_metricCollectionFactory.GetMetrics(),
_collectDefaultXUnitMetrics);
if (IsWindowsPlatform && _requireEtw)
{
void winRunner()
{
xUnitAction(assemblyFileName);
}
ETWProfiler.Record(
xUnitPerformanceSessionData,
xUnitPerformanceMetricData,
winRunner);
}
else
{
xUnitAction(assemblyFileName);
ProcessResults(xUnitPerformanceSessionData, xUnitPerformanceMetricData);
}
}
public void RunBenchmarks(string assemblyFileName)
{
Validate(assemblyFileName);
Action <string> xUnitAction = (assemblyPath) => { XunitRunner.Run(assemblyPath, _typeNames); };
var xUnitPerformanceSessionData = new XUnitPerformanceSessionData {
AssemblyFileName = assemblyFileName,
CollectOutputFilesCallback = (fileName) => {
// FIXME: This will need safe guards when the client calls RunBenchmarks in different threads.
_outputFiles.Add(fileName);
WriteInfoLine($"File saved to: \"{fileName}\"");
},
OutputDirectory = OutputDirectory,
RunId = Configuration.RunId
};
var metrics = _metricCollectionFactory.GetMetrics(assemblyFileName);
var xUnitPerformanceMetricData = XunitBenchmark.GetMetadata(
assemblyFileName,
metrics,
_collectDefaultXUnitMetrics);
if (IsWindowsPlatform && _requireEtw)
{
Action winRunner = () => { xUnitAction(assemblyFileName); };
ETWProfiler.Record(
xUnitPerformanceSessionData,
xUnitPerformanceMetricData,
winRunner);
}
else
{
xUnitAction.Invoke(assemblyFileName);
ProcessResults(xUnitPerformanceSessionData, xUnitPerformanceMetricData);
}
}
/// <summary>
/// 1. In the specified assembly, get the ETW providers set as assembly attributes (PerformanceTestInfo)
/// 2. Check if the benchmark assembly request Precise Machine Counters(PMC) to be collected
/// 3. Enable Kernel providers if needed
/// 4. Get non-kernel ETW flags set and enable them
/// 5. Run the benchmarks
/// 6. Stop collecting ETW
/// 7. Merge ETL files.
/// </summary>
/// <param name="xUnitPerformanceSessionData"></param>
/// <param name="xUnitPerformanceMetricData"></param>
/// <param name="action"></param>
public static void Record(XUnitPerformanceSessionData xUnitPerformanceSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData, Action action)
{
const int bufferSizeMB = 256;
var name = $"{xUnitPerformanceSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitPerformanceSessionData.AssemblyFileName)}";
var etwOutputData = new ETWOutputData {
KernelFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.kernel.etl"), // without this parameter, EnableKernelProvider will fail
Name = name,
SessionName = $"Performance-Api-Session-{xUnitPerformanceSessionData.RunId}",
UserFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.etl"),
};
PrintProfilingInformation(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData);
var kernelProviderInfo = xUnitPerformanceMetricData.Providers.OfType <KernelProviderInfo>().FirstOrDefault();
var needKernelSession = NeedSeparateKernelSession(kernelProviderInfo);
if (needKernelSession && !CanEnableEnableKernelProvider)
{
const string message = "In order to capture kernel data the application is required to run as Administrator.";
WriteErrorLine(message);
throw new InvalidOperationException(message);
}
WriteDebugLine("> ETW capture start.");
// Prior to Windows 8 (NT 6.2), all kernel events needed the special kernel session.
using (var safeKernelSession = needKernelSession && CanEnableEnableKernelProvider ? MakeSafeTerminateTraceEventSession(KernelTraceEventParser.KernelSessionName, etwOutputData.KernelFileName) : null)
{
var kernelSession = safeKernelSession?.BaseDisposableObject;
if (kernelSession != null)
{
SetPreciseMachineCounters(xUnitPerformanceMetricData.Providers);
kernelSession.BufferSizeMB = bufferSizeMB;
var flags = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords;
var stackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords;
kernelSession.EnableKernelProvider(flags, stackCapture);
}
using (var safeUserEventSession = MakeSafeTerminateTraceEventSession(etwOutputData.SessionName, etwOutputData.UserFileName))
{
var userEventSession = safeUserEventSession.BaseDisposableObject;
userEventSession.BufferSizeMB = bufferSizeMB;
if (needKernelSession && CanEnableEnableKernelProvider)
{
userEventSession.EnableKernelProvider(KernelProvider.Default.Flags, KernelProvider.Default.StackCapture);
}
foreach (var provider in UserProvider.Defaults)
{
userEventSession.EnableProvider(provider.Guid, provider.Level, provider.Keywords);
}
foreach (var provider in xUnitPerformanceMetricData.Providers.OfType <UserProviderInfo>())
{
userEventSession.EnableProvider(provider.ProviderGuid, provider.Level, provider.Keywords);
}
action.Invoke();
}
}
WriteDebugLine("> ETW capture stop.");
// TODO: Decouple the code below.
// Collect ETW profile data.
// TODO: Skip collecting kernel data if it was not captured! (data will be zero, there is no point to report it or upload it)
WriteDebugLine("> ETW merge start.");
TraceEventSession.MergeInPlace(etwOutputData.UserFileName, Console.Out);
WriteDebugLine("> ETW merge stop.");
xUnitPerformanceSessionData.CollectOutputFilesCallback(etwOutputData.UserFileName);
var assemblyModel = GetAssemblyModel(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData.UserFileName, xUnitPerformanceSessionData.RunId, xUnitPerformanceMetricData.PerformanceTestMessages);
var xmlFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.xml");
new AssemblyModelCollection {
assemblyModel
}.Serialize(xmlFileName);
xUnitPerformanceSessionData.CollectOutputFilesCallback(xmlFileName);
var mdFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.md");
var dt = assemblyModel.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
MarkdownHelper.Write(mdFileName, mdTable);
xUnitPerformanceSessionData.CollectOutputFilesCallback(mdFileName);
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
var csvFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.csv");
dt.WriteToCSV(csvFileName);
xUnitPerformanceSessionData.CollectOutputFilesCallback(csvFileName);
}
