/// <summary>
/// Run the xUnit tests tagged with the [<see cref="BenchmarkAttribute"/>] attribute.
/// </summary>
/// <param name="assemblyFileName">Path to the assembly that contains the xUnit performance tests.</param>
public void RunBenchmarks(string assemblyFileName)
{
if (string.IsNullOrEmpty(assemblyFileName))
{
throw new ArgumentNullException(nameof(assemblyFileName));
}
if (!File.Exists(assemblyFileName))
{
throw new FileNotFoundException(assemblyFileName);
}
void xUnitAction(string assemblyPath)
{
var errCode = XunitRunner.Run(assemblyPath, _typeNames);
if (errCode != 0)
{
throw new Exception($"{errCode} benchmark(s) failed to execute.");
}
}
var xUnitPerformanceSessionData = new XUnitPerformanceSessionData
{
AssemblyFileName = assemblyFileName,
CollectOutputFilesCallback = LogFileSaved,
OutputDirectory = OutputDirectory,
RunId = Configuration.RunId
};
var xUnitPerformanceMetricData = XunitBenchmark.GetMetadata(
assemblyFileName,
_metricCollectionFactory.GetMetrics(),
_collectDefaultXUnitMetrics);
if (IsWindowsPlatform && _requireEtw)
{
void winRunner()
{
xUnitAction(assemblyFileName);
}
ETWProfiler.Record(
xUnitPerformanceSessionData,
xUnitPerformanceMetricData,
winRunner);
}
else
{
xUnitAction(assemblyFileName);
ProcessResults(xUnitPerformanceSessionData, xUnitPerformanceMetricData);
}
}
private static IEnumerable <(PerformanceTestMessage performanceTestMessage, string metric, IEnumerable <double> values)> GetCollectedData(string assemblyFileName, CSVMetricReader reader)
{
var testsFoundInAssembly = (XunitBenchmark.GetMetadata(assemblyFileName)).performanceTestMessages;
foreach (var testFoundInAssembly in testsFoundInAssembly)
{
foreach (var(testCaseName, metric, values) in GetMeasurements(reader))
{
if (values == null)
{
continue;
}
if (testCaseName == testFoundInAssembly.TestCase.DisplayName)
{
yield return(testFoundInAssembly, metric, values);
}
}
}
}
public void RunBenchmarks(string assemblyFileName)
{
Validate(assemblyFileName);
Action <string> xUnitAction = (assemblyPath) => { XunitRunner.Run(assemblyPath, _typeNames); };
var xUnitPerformanceSessionData = new XUnitPerformanceSessionData {
AssemblyFileName = assemblyFileName,
CollectOutputFilesCallback = (fileName) => {
// FIXME: This will need safe guards when the client calls RunBenchmarks in different threads.
_outputFiles.Add(fileName);
WriteInfoLine($"File saved to: \"{fileName}\"");
},
OutputDirectory = OutputDirectory,
RunId = Configuration.RunId
};
var metrics = _metricCollectionFactory.GetMetrics(assemblyFileName);
var xUnitPerformanceMetricData = XunitBenchmark.GetMetadata(
assemblyFileName,
metrics,
_collectDefaultXUnitMetrics);
if (IsWindowsPlatform && _requireEtw)
{
Action winRunner = () => { xUnitAction(assemblyFileName); };
ETWProfiler.Record(
xUnitPerformanceSessionData,
xUnitPerformanceMetricData,
winRunner);
}
else
{
xUnitAction.Invoke(assemblyFileName);
ProcessResults(xUnitPerformanceSessionData, xUnitPerformanceMetricData);
}
}
/// <summary>
/// 1. In the specified assembly, get the ETW providers set as assembly attributes (PerformanceTestInfo)
/// 2. Check if the benchmark assembly request Precise Machine Counters(PMC) to be collected
/// 3. Enable Kernel providers if needed
/// 4. Get non-kernel ETW flags set and enable them
/// 5. Run the benchmarks
/// 6. Stop collecting ETW
/// 7. Merge ETL files.
/// </summary>
/// <param name="assemblyFileName"></param>
/// <param name="runId"></param>
/// <param name="outputDirectory"></param>
/// <param name="action"></param>
/// <param name="collectOutputFilesCallback">Callback used to collect a list of files generated.</param>
/// <returns></returns>
public static void Record(string assemblyFileName, string runId, string outputDirectory, Action action, Action <string> collectOutputFilesCallback)
{
if (TraceEventSession.IsElevated() != true)
{
const string errMessage = "In order to profile, application is required to run as Administrator.";
WriteErrorLine(errMessage);
throw new InvalidOperationException(errMessage);
}
const int bufferSizeMB = 256;
var sessionName = $"Performance-Api-Session-{runId}";
var name = $"{runId}-{Path.GetFileNameWithoutExtension(assemblyFileName)}";
var userFullFileName = Path.Combine(outputDirectory, $"{name}.etl");
var kernelFullFileName = Path.Combine(outputDirectory, $"{name}.kernel.etl"); // without this parameter, EnableKernelProvider will fail
PrintProfilingInformation(assemblyFileName, sessionName, userFullFileName);
(var providers, var performanceTestMessages) = XunitBenchmark.GetMetadata(assemblyFileName);
var kernelProviderInfo = providers.OfType <KernelProviderInfo>().FirstOrDefault();
var needKernelSession = NeedSeparateKernelSession(kernelProviderInfo);
using (var safeKernelSession = needKernelSession ? MakeSafeTerminateTraceEventSession(KernelTraceEventParser.KernelSessionName, kernelFullFileName) : null)
{
var kernelSession = safeKernelSession?.BaseDisposableObject;
if (kernelSession != null)
{
SetPreciseMachineCounters(providers);
kernelSession.BufferSizeMB = bufferSizeMB;
var flags = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords;
var stackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords;
kernelSession.EnableKernelProvider(flags, stackCapture);
}
using (var safeUserEventSession = MakeSafeTerminateTraceEventSession(sessionName, userFullFileName))
{
var userEventSession = safeUserEventSession.BaseDisposableObject;
userEventSession.BufferSizeMB = bufferSizeMB;
var flags = KernelTraceEventParser.Keywords.Process | KernelTraceEventParser.Keywords.ImageLoad | KernelTraceEventParser.Keywords.Thread;
var stackCapture = KernelTraceEventParser.Keywords.Profile | KernelTraceEventParser.Keywords.ContextSwitch;
userEventSession.EnableKernelProvider(flags, stackCapture);
foreach (var userProviderInfo in providers.OfType <UserProviderInfo>())
{
userEventSession.EnableProvider(userProviderInfo.ProviderGuid, userProviderInfo.Level, userProviderInfo.Keywords);
}
action.Invoke();
}
}
TraceEventSession.MergeInPlace(userFullFileName, Console.Out);
WriteInfoLine($"ETW Tracing Session saved to \"{userFullFileName}\"");
collectOutputFilesCallback(userFullFileName);
var assemblyModel = GetAssemblyModel(assemblyFileName, userFullFileName, runId, performanceTestMessages);
var xmlFileName = Path.Combine(outputDirectory, $"{name}.xml");
new AssemblyModelCollection {
assemblyModel
}.Serialize(xmlFileName);
WriteInfoLine($"Performance results saved to \"{xmlFileName}\"");
collectOutputFilesCallback(xmlFileName);
var mdFileName = Path.Combine(outputDirectory, $"{name}.md");
var dt = assemblyModel.GetStatistics();
var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);
MarkdownHelper.Write(mdFileName, mdTable);
WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
collectOutputFilesCallback(mdFileName);
Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
var csvFileName = Path.Combine(outputDirectory, $"{name}.csv");
dt.WriteToCSV(csvFileName);
WriteInfoLine($"Statistics written to \"{csvFileName}\"");
collectOutputFilesCallback(csvFileName);
}
