/// <summary> /// Run the xUnit tests tagged with the [<see cref="BenchmarkAttribute"/>] attribute. /// </summary> /// <param name="assemblyFileName">Path to the assembly that contains the xUnit performance tests.</param> public void RunBenchmarks(string assemblyFileName) { if (string.IsNullOrEmpty(assemblyFileName)) { throw new ArgumentNullException(nameof(assemblyFileName)); } if (!File.Exists(assemblyFileName)) { throw new FileNotFoundException(assemblyFileName); } void xUnitAction(string assemblyPath) { var errCode = XunitRunner.Run(assemblyPath, _typeNames); if (errCode != 0) { throw new Exception($"{errCode} benchmark(s) failed to execute."); } } var xUnitPerformanceSessionData = new XUnitPerformanceSessionData { AssemblyFileName = assemblyFileName, CollectOutputFilesCallback = LogFileSaved, OutputDirectory = OutputDirectory, RunId = Configuration.RunId }; var xUnitPerformanceMetricData = XunitBenchmark.GetMetadata( assemblyFileName, _metricCollectionFactory.GetMetrics(), _collectDefaultXUnitMetrics); if (IsWindowsPlatform && _requireEtw) { void winRunner() { xUnitAction(assemblyFileName); } ETWProfiler.Record( xUnitPerformanceSessionData, xUnitPerformanceMetricData, winRunner); } else { xUnitAction(assemblyFileName); ProcessResults(xUnitPerformanceSessionData, xUnitPerformanceMetricData); } }
private static IEnumerable <(PerformanceTestMessage performanceTestMessage, string metric, IEnumerable <double> values)> GetCollectedData(string assemblyFileName, CSVMetricReader reader) { var testsFoundInAssembly = (XunitBenchmark.GetMetadata(assemblyFileName)).performanceTestMessages; foreach (var testFoundInAssembly in testsFoundInAssembly) { foreach (var(testCaseName, metric, values) in GetMeasurements(reader)) { if (values == null) { continue; } if (testCaseName == testFoundInAssembly.TestCase.DisplayName) { yield return(testFoundInAssembly, metric, values); } } } }
public void RunBenchmarks(string assemblyFileName) { Validate(assemblyFileName); Action <string> xUnitAction = (assemblyPath) => { XunitRunner.Run(assemblyPath, _typeNames); }; var xUnitPerformanceSessionData = new XUnitPerformanceSessionData { AssemblyFileName = assemblyFileName, CollectOutputFilesCallback = (fileName) => { // FIXME: This will need safe guards when the client calls RunBenchmarks in different threads. _outputFiles.Add(fileName); WriteInfoLine($"File saved to: \"{fileName}\""); }, OutputDirectory = OutputDirectory, RunId = Configuration.RunId }; var metrics = _metricCollectionFactory.GetMetrics(assemblyFileName); var xUnitPerformanceMetricData = XunitBenchmark.GetMetadata( assemblyFileName, metrics, _collectDefaultXUnitMetrics); if (IsWindowsPlatform && _requireEtw) { Action winRunner = () => { xUnitAction(assemblyFileName); }; ETWProfiler.Record( xUnitPerformanceSessionData, xUnitPerformanceMetricData, winRunner); } else { xUnitAction.Invoke(assemblyFileName); ProcessResults(xUnitPerformanceSessionData, xUnitPerformanceMetricData); } }
/// <summary> /// 1. In the specified assembly, get the ETW providers set as assembly attributes (PerformanceTestInfo) /// 2. Check if the benchmark assembly request Precise Machine Counters(PMC) to be collected /// 3. Enable Kernel providers if needed /// 4. Get non-kernel ETW flags set and enable them /// 5. Run the benchmarks /// 6. Stop collecting ETW /// 7. Merge ETL files. /// </summary> /// <param name="assemblyFileName"></param> /// <param name="runId"></param> /// <param name="outputDirectory"></param> /// <param name="action"></param> /// <param name="collectOutputFilesCallback">Callback used to collect a list of files generated.</param> /// <returns></returns> public static void Record(string assemblyFileName, string runId, string outputDirectory, Action action, Action <string> collectOutputFilesCallback) { if (TraceEventSession.IsElevated() != true) { const string errMessage = "In order to profile, application is required to run as Administrator."; WriteErrorLine(errMessage); throw new InvalidOperationException(errMessage); } const int bufferSizeMB = 256; var sessionName = $"Performance-Api-Session-{runId}"; var name = $"{runId}-{Path.GetFileNameWithoutExtension(assemblyFileName)}"; var userFullFileName = Path.Combine(outputDirectory, $"{name}.etl"); var kernelFullFileName = Path.Combine(outputDirectory, $"{name}.kernel.etl"); // without this parameter, EnableKernelProvider will fail PrintProfilingInformation(assemblyFileName, sessionName, userFullFileName); (var providers, var performanceTestMessages) = XunitBenchmark.GetMetadata(assemblyFileName); var kernelProviderInfo = providers.OfType <KernelProviderInfo>().FirstOrDefault(); var needKernelSession = NeedSeparateKernelSession(kernelProviderInfo); using (var safeKernelSession = needKernelSession ? MakeSafeTerminateTraceEventSession(KernelTraceEventParser.KernelSessionName, kernelFullFileName) : null) { var kernelSession = safeKernelSession?.BaseDisposableObject; if (kernelSession != null) { SetPreciseMachineCounters(providers); kernelSession.BufferSizeMB = bufferSizeMB; var flags = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords; var stackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords; kernelSession.EnableKernelProvider(flags, stackCapture); } using (var safeUserEventSession = MakeSafeTerminateTraceEventSession(sessionName, userFullFileName)) { var userEventSession = safeUserEventSession.BaseDisposableObject; userEventSession.BufferSizeMB = bufferSizeMB; var flags = KernelTraceEventParser.Keywords.Process | KernelTraceEventParser.Keywords.ImageLoad | KernelTraceEventParser.Keywords.Thread; var stackCapture = KernelTraceEventParser.Keywords.Profile | KernelTraceEventParser.Keywords.ContextSwitch; userEventSession.EnableKernelProvider(flags, stackCapture); foreach (var userProviderInfo in providers.OfType <UserProviderInfo>()) { userEventSession.EnableProvider(userProviderInfo.ProviderGuid, userProviderInfo.Level, userProviderInfo.Keywords); } action.Invoke(); } } TraceEventSession.MergeInPlace(userFullFileName, Console.Out); WriteInfoLine($"ETW Tracing Session saved to \"{userFullFileName}\""); collectOutputFilesCallback(userFullFileName); var assemblyModel = GetAssemblyModel(assemblyFileName, userFullFileName, runId, performanceTestMessages); var xmlFileName = Path.Combine(outputDirectory, $"{name}.xml"); new AssemblyModelCollection { assemblyModel }.Serialize(xmlFileName); WriteInfoLine($"Performance results saved to \"{xmlFileName}\""); collectOutputFilesCallback(xmlFileName); var mdFileName = Path.Combine(outputDirectory, $"{name}.md"); var dt = assemblyModel.GetStatistics(); var mdTable = MarkdownHelper.GenerateMarkdownTable(dt); MarkdownHelper.Write(mdFileName, mdTable); WriteInfoLine($"Markdown file saved to \"{mdFileName}\""); collectOutputFilesCallback(mdFileName); Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable)); var csvFileName = Path.Combine(outputDirectory, $"{name}.csv"); dt.WriteToCSV(csvFileName); WriteInfoLine($"Statistics written to \"{csvFileName}\""); collectOutputFilesCallback(csvFileName); }