private void ProcessResults(XUnitPerformanceSessionData xUnitSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData) { if (!File.Exists(Configuration.FileLogPath)) { WriteWarningLine($"Results file '{Configuration.FileLogPath}' does not exist. Skipping processing of results."); return; } var reader = new CSVMetricReader(Configuration.FileLogPath); var fileNameWithoutExtension = $"{xUnitSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitSessionData.AssemblyFileName)}"; var assemblyModel = AssemblyModel.Create(xUnitSessionData.AssemblyFileName, reader, xUnitPerformanceMetricData); var xmlFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.xml"); new AssemblyModelCollection { assemblyModel }.Serialize(xmlFileName); xUnitSessionData.CollectOutputFilesCallback(xmlFileName); var dt = assemblyModel.GetStatistics(); var mdTable = MarkdownHelper.GenerateMarkdownTable(dt); var mdFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.md"); MarkdownHelper.Write(mdFileName, mdTable); xUnitSessionData.CollectOutputFilesCallback(mdFileName); Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable)); var csvFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.csv"); dt.WriteToCSV(csvFileName); xUnitSessionData.CollectOutputFilesCallback(csvFileName); BenchmarkEventSource.Log.Clear(); }
/// <summary> /// Run the xUnit tests tagged with the [<see cref="BenchmarkAttribute"/>] attribute. /// </summary> /// <param name="assemblyFileName">Path to the assembly that contains the xUnit performance tests.</param> public void RunBenchmarks(string assemblyFileName) { if (string.IsNullOrEmpty(assemblyFileName)) { throw new ArgumentNullException(nameof(assemblyFileName)); } if (!File.Exists(assemblyFileName)) { throw new FileNotFoundException(assemblyFileName); } void xUnitAction(string assemblyPath) { var errCode = XunitRunner.Run(assemblyPath, _typeNames); if (errCode != 0) { throw new Exception($"{errCode} benchmark(s) failed to execute."); } } var xUnitPerformanceSessionData = new XUnitPerformanceSessionData { AssemblyFileName = assemblyFileName, CollectOutputFilesCallback = LogFileSaved, OutputDirectory = OutputDirectory, RunId = Configuration.RunId }; var xUnitPerformanceMetricData = XunitBenchmark.GetMetadata( assemblyFileName, _metricCollectionFactory.GetMetrics(), _collectDefaultXUnitMetrics); if (IsWindowsPlatform && _requireEtw) { void winRunner() { xUnitAction(assemblyFileName); } ETWProfiler.Record( xUnitPerformanceSessionData, xUnitPerformanceMetricData, winRunner); } else { xUnitAction(assemblyFileName); ProcessResults(xUnitPerformanceSessionData, xUnitPerformanceMetricData); } }
public void RunBenchmarks(string assemblyFileName) { Validate(assemblyFileName); Action <string> xUnitAction = (assemblyPath) => { XunitRunner.Run(assemblyPath, _typeNames); }; var xUnitPerformanceSessionData = new XUnitPerformanceSessionData { AssemblyFileName = assemblyFileName, CollectOutputFilesCallback = (fileName) => { // FIXME: This will need safe guards when the client calls RunBenchmarks in different threads. _outputFiles.Add(fileName); WriteInfoLine($"File saved to: \"{fileName}\""); }, OutputDirectory = OutputDirectory, RunId = Configuration.RunId }; var metrics = _metricCollectionFactory.GetMetrics(assemblyFileName); var xUnitPerformanceMetricData = XunitBenchmark.GetMetadata( assemblyFileName, metrics, _collectDefaultXUnitMetrics); if (IsWindowsPlatform && _requireEtw) { Action winRunner = () => { xUnitAction(assemblyFileName); }; ETWProfiler.Record( xUnitPerformanceSessionData, xUnitPerformanceMetricData, winRunner); } else { xUnitAction.Invoke(assemblyFileName); ProcessResults(xUnitPerformanceSessionData, xUnitPerformanceMetricData); } }
/// <summary> /// 1. In the specified assembly, get the ETW providers set as assembly attributes (PerformanceTestInfo) /// 2. Check if the benchmark assembly request Precise Machine Counters(PMC) to be collected /// 3. Enable Kernel providers if needed /// 4. Get non-kernel ETW flags set and enable them /// 5. Run the benchmarks /// 6. Stop collecting ETW /// 7. Merge ETL files. /// </summary> /// <param name="xUnitPerformanceSessionData"></param> /// <param name="xUnitPerformanceMetricData"></param> /// <param name="action"></param> public static void Record(XUnitPerformanceSessionData xUnitPerformanceSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData, Action action) { const int bufferSizeMB = 256; var name = $"{xUnitPerformanceSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitPerformanceSessionData.AssemblyFileName)}"; var etwOutputData = new ETWOutputData { KernelFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.kernel.etl"), // without this parameter, EnableKernelProvider will fail Name = name, SessionName = $"Performance-Api-Session-{xUnitPerformanceSessionData.RunId}", UserFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.etl"), }; PrintProfilingInformation(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData); var kernelProviderInfo = xUnitPerformanceMetricData.Providers.OfType <KernelProviderInfo>().FirstOrDefault(); var needKernelSession = NeedSeparateKernelSession(kernelProviderInfo); if (needKernelSession && !CanEnableEnableKernelProvider) { const string message = "In order to capture kernel data the application is required to run as Administrator."; WriteErrorLine(message); throw new InvalidOperationException(message); } WriteDebugLine("> ETW capture start."); // Prior to Windows 8 (NT 6.2), all kernel events needed the special kernel session. using (var safeKernelSession = needKernelSession && CanEnableEnableKernelProvider ? MakeSafeTerminateTraceEventSession(KernelTraceEventParser.KernelSessionName, etwOutputData.KernelFileName) : null) { var kernelSession = safeKernelSession?.BaseDisposableObject; if (kernelSession != null) { SetPreciseMachineCounters(xUnitPerformanceMetricData.Providers); kernelSession.BufferSizeMB = bufferSizeMB; var flags = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords; var stackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords; kernelSession.EnableKernelProvider(flags, stackCapture); } using (var safeUserEventSession = MakeSafeTerminateTraceEventSession(etwOutputData.SessionName, etwOutputData.UserFileName)) { var userEventSession = safeUserEventSession.BaseDisposableObject; userEventSession.BufferSizeMB = bufferSizeMB; if (needKernelSession && CanEnableEnableKernelProvider) { userEventSession.EnableKernelProvider(KernelProvider.Default.Flags, KernelProvider.Default.StackCapture); } foreach (var provider in UserProvider.Defaults) { userEventSession.EnableProvider(provider.Guid, provider.Level, provider.Keywords); } foreach (var provider in xUnitPerformanceMetricData.Providers.OfType <UserProviderInfo>()) { userEventSession.EnableProvider(provider.ProviderGuid, provider.Level, provider.Keywords); } action.Invoke(); } } WriteDebugLine("> ETW capture stop."); // TODO: Decouple the code below. // Collect ETW profile data. // TODO: Skip collecting kernel data if it was not captured! (data will be zero, there is no point to report it or upload it) WriteDebugLine("> ETW merge start."); TraceEventSession.MergeInPlace(etwOutputData.UserFileName, Console.Out); WriteDebugLine("> ETW merge stop."); xUnitPerformanceSessionData.CollectOutputFilesCallback(etwOutputData.UserFileName); var assemblyModel = GetAssemblyModel(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData.UserFileName, xUnitPerformanceSessionData.RunId, xUnitPerformanceMetricData.PerformanceTestMessages); var xmlFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.xml"); new AssemblyModelCollection { assemblyModel }.Serialize(xmlFileName); xUnitPerformanceSessionData.CollectOutputFilesCallback(xmlFileName); var mdFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.md"); var dt = assemblyModel.GetStatistics(); var mdTable = MarkdownHelper.GenerateMarkdownTable(dt); MarkdownHelper.Write(mdFileName, mdTable); xUnitPerformanceSessionData.CollectOutputFilesCallback(mdFileName); Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable)); var csvFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.csv"); dt.WriteToCSV(csvFileName); xUnitPerformanceSessionData.CollectOutputFilesCallback(csvFileName); }