private void ProcessResults(XUnitPerformanceSessionData xUnitSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData)
{
if (!File.Exists(Configuration.FileLogPath))
{
WriteWarningLine($"Results file '{Configuration.FileLogPath}' does not exist. Skipping processing of results.");
return;
}
var reader = new CSVMetricReader(Configuration.FileLogPath);
var fileNameWithoutExtension = $"{xUnitSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitSessionData.AssemblyFileName)}";
var assemblyModel = AssemblyModel.Create(xUnitSessionData.AssemblyFileName, reader, xUnitPerformanceMetricData);
var xmlFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.xml");
new AssemblyModelCollection {
assemblyModel
}.Serialize(xmlFileName);
xUnitSessionData.CollectOutputFilesCallback(xmlFileName);
var dt = assemblyModel.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
var mdFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.md");
MarkdownHelper.Write(mdFileName, mdTable);
xUnitSessionData.CollectOutputFilesCallback(mdFileName);
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
var csvFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.csv");
dt.WriteToCSV(csvFileName);
xUnitSessionData.CollectOutputFilesCallback(csvFileName);
BenchmarkEventSource.Log.Clear();
}
internal static AssemblyModel Create(
string assemblyFileName,
CSVMetricReader reader,
XUnitPerformanceMetricData xUnitPerformanceMetricData)
{
var assemblyModel = new AssemblyModel
{
Name = Path.GetFileName(assemblyFileName),
Collection = new List <TestModel>()
};
foreach (var(perfTestMsg, metric, values) in GetCollectedData(assemblyFileName, reader, xUnitPerformanceMetricData))
{
var testModel = assemblyModel.Collection.FirstOrDefault(test => test.Name == perfTestMsg.TestCase.DisplayName);
if (testModel == null)
{
testModel = new TestModel
{
Name = perfTestMsg.TestCase.DisplayName,
Method = perfTestMsg.TestCase.TestMethod.Method.Name,
ClassName = perfTestMsg.TestCase.TestMethod.TestClass.Class.Name,
Performance = new PerformanceModel
{
Metrics = new List <MetricModel>(),
IterationModels = new List <IterationModel>()
},
};
}
var testMetric = testModel.Performance.Metrics.FirstOrDefault(m => m.DisplayName == metric);
if (testMetric == null)
{
testModel.Performance.Metrics.Add(new MetricModel
{
DisplayName = metric,
Name = metric,
Unit = metric == "Duration" ? PerformanceMetricUnits.Milliseconds : "unknown", // We are guessing here.
});
}
foreach (var value in values)
{
var iterationModel = new IterationModel {
Iteration = new Dictionary <string, double>()
};
iterationModel.Iteration.Add(metric, value);
if (iterationModel.Iteration.Count > 0)
{
testModel.Performance.IterationModels.Add(iterationModel);
}
}
assemblyModel.Collection.Add(testModel);
}
return(assemblyModel);
}
private static IEnumerable <(PerformanceTestMessage performanceTestMessage, string metric, IEnumerable <double> values)> GetCollectedData(
string assemblyFileName,
CSVMetricReader reader,
XUnitPerformanceMetricData xUnitPerformanceMetricData)
{
var testsFoundInAssembly = xUnitPerformanceMetricData.PerformanceTestMessages;
foreach (var testFoundInAssembly in testsFoundInAssembly)
{
foreach (var(testCaseName, metric, values) in GetMeasurements(reader))
{
if (values == null)
{
continue;
}
if (testCaseName == testFoundInAssembly.TestCase.DisplayName)
{
yield return(testFoundInAssembly, metric, values);
}
}
}
}
/// <summary>
/// 1. In the specified assembly, get the ETW providers set as assembly attributes (PerformanceTestInfo)
/// 2. Check if the benchmark assembly request Precise Machine Counters(PMC) to be collected
/// 3. Enable Kernel providers if needed
/// 4. Get non-kernel ETW flags set and enable them
/// 5. Run the benchmarks
/// 6. Stop collecting ETW
/// 7. Merge ETL files.
/// </summary>
/// <param name="xUnitPerformanceSessionData"></param>
/// <param name="xUnitPerformanceMetricData"></param>
/// <param name="action"></param>
public static void Record(XUnitPerformanceSessionData xUnitPerformanceSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData, Action action)
{
const int bufferSizeMB = 256;
var name = $"{xUnitPerformanceSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitPerformanceSessionData.AssemblyFileName)}";
var etwOutputData = new ETWOutputData {
KernelFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.kernel.etl"), // without this parameter, EnableKernelProvider will fail
Name = name,
SessionName = $"Performance-Api-Session-{xUnitPerformanceSessionData.RunId}",
UserFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.etl"),
};
PrintProfilingInformation(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData);
var kernelProviderInfo = xUnitPerformanceMetricData.Providers.OfType <KernelProviderInfo>().FirstOrDefault();
var needKernelSession = NeedSeparateKernelSession(kernelProviderInfo);
if (needKernelSession && !CanEnableEnableKernelProvider)
{
const string message = "In order to capture kernel data the application is required to run as Administrator.";
WriteErrorLine(message);
throw new InvalidOperationException(message);
}
WriteDebugLine("> ETW capture start.");
// Prior to Windows 8 (NT 6.2), all kernel events needed the special kernel session.
using (var safeKernelSession = needKernelSession && CanEnableEnableKernelProvider ? MakeSafeTerminateTraceEventSession(KernelTraceEventParser.KernelSessionName, etwOutputData.KernelFileName) : null)
{
var kernelSession = safeKernelSession?.BaseDisposableObject;
if (kernelSession != null)
{
SetPreciseMachineCounters(xUnitPerformanceMetricData.Providers);
kernelSession.BufferSizeMB = bufferSizeMB;
var flags = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords;
var stackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords;
kernelSession.EnableKernelProvider(flags, stackCapture);
}
using (var safeUserEventSession = MakeSafeTerminateTraceEventSession(etwOutputData.SessionName, etwOutputData.UserFileName))
{
var userEventSession = safeUserEventSession.BaseDisposableObject;
userEventSession.BufferSizeMB = bufferSizeMB;
if (needKernelSession && CanEnableEnableKernelProvider)
{
userEventSession.EnableKernelProvider(KernelProvider.Default.Flags, KernelProvider.Default.StackCapture);
}
foreach (var provider in UserProvider.Defaults)
{
userEventSession.EnableProvider(provider.Guid, provider.Level, provider.Keywords);
}
foreach (var provider in xUnitPerformanceMetricData.Providers.OfType <UserProviderInfo>())
{
userEventSession.EnableProvider(provider.ProviderGuid, provider.Level, provider.Keywords);
}
action.Invoke();
}
}
WriteDebugLine("> ETW capture stop.");
// TODO: Decouple the code below.
// Collect ETW profile data.
// TODO: Skip collecting kernel data if it was not captured! (data will be zero, there is no point to report it or upload it)
WriteDebugLine("> ETW merge start.");
TraceEventSession.MergeInPlace(etwOutputData.UserFileName, Console.Out);
WriteDebugLine("> ETW merge stop.");
xUnitPerformanceSessionData.CollectOutputFilesCallback(etwOutputData.UserFileName);
var assemblyModel = GetAssemblyModel(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData.UserFileName, xUnitPerformanceSessionData.RunId, xUnitPerformanceMetricData.PerformanceTestMessages);
var xmlFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.xml");
new AssemblyModelCollection {
assemblyModel
}.Serialize(xmlFileName);
xUnitPerformanceSessionData.CollectOutputFilesCallback(xmlFileName);
var mdFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.md");
var dt = assemblyModel.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
MarkdownHelper.Write(mdFileName, mdTable);
xUnitPerformanceSessionData.CollectOutputFilesCallback(mdFileName);
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
var csvFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.csv");
dt.WriteToCSV(csvFileName);
xUnitPerformanceSessionData.CollectOutputFilesCallback(csvFileName);
}