示例#1
0
        private void ProcessResults(XUnitPerformanceSessionData xUnitSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData)
        {
            if (!File.Exists(Configuration.FileLogPath))
            {
                WriteWarningLine($"Results file '{Configuration.FileLogPath}' does not exist. Skipping processing of results.");
                return;
            }

            var reader = new CSVMetricReader(Configuration.FileLogPath);
            var fileNameWithoutExtension = $"{xUnitSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitSessionData.AssemblyFileName)}";

            var assemblyModel = AssemblyModel.Create(xUnitSessionData.AssemblyFileName, reader, xUnitPerformanceMetricData);
            var xmlFileName   = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.xml");

            new AssemblyModelCollection {
                assemblyModel
            }.Serialize(xmlFileName);
            xUnitSessionData.CollectOutputFilesCallback(xmlFileName);

            var dt         = assemblyModel.GetStatistics();
            var mdTable    = MarkdownHelper.GenerateMarkdownTable(dt);
            var mdFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.md");

            MarkdownHelper.Write(mdFileName, mdTable);
            xUnitSessionData.CollectOutputFilesCallback(mdFileName);
            Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));

            var csvFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.csv");

            dt.WriteToCSV(csvFileName);
            xUnitSessionData.CollectOutputFilesCallback(csvFileName);

            BenchmarkEventSource.Log.Clear();
        }
示例#2
0
        internal static AssemblyModel Create(
            string assemblyFileName,
            CSVMetricReader reader,
            XUnitPerformanceMetricData xUnitPerformanceMetricData)
        {
            var assemblyModel = new AssemblyModel
            {
                Name       = Path.GetFileName(assemblyFileName),
                Collection = new List <TestModel>()
            };

            foreach (var(perfTestMsg, metric, values) in GetCollectedData(assemblyFileName, reader, xUnitPerformanceMetricData))
            {
                var testModel = assemblyModel.Collection.FirstOrDefault(test => test.Name == perfTestMsg.TestCase.DisplayName);
                if (testModel == null)
                {
                    testModel = new TestModel
                    {
                        Name        = perfTestMsg.TestCase.DisplayName,
                        Method      = perfTestMsg.TestCase.TestMethod.Method.Name,
                        ClassName   = perfTestMsg.TestCase.TestMethod.TestClass.Class.Name,
                        Performance = new PerformanceModel
                        {
                            Metrics         = new List <MetricModel>(),
                            IterationModels = new List <IterationModel>()
                        },
                    };
                }

                var testMetric = testModel.Performance.Metrics.FirstOrDefault(m => m.DisplayName == metric);
                if (testMetric == null)
                {
                    testModel.Performance.Metrics.Add(new MetricModel
                    {
                        DisplayName = metric,
                        Name        = metric,
                        Unit        = metric == "Duration" ? PerformanceMetricUnits.Milliseconds : "unknown", // We are guessing here.
                    });
                }

                foreach (var value in values)
                {
                    var iterationModel = new IterationModel {
                        Iteration = new Dictionary <string, double>()
                    };
                    iterationModel.Iteration.Add(metric, value);
                    if (iterationModel.Iteration.Count > 0)
                    {
                        testModel.Performance.IterationModels.Add(iterationModel);
                    }
                }

                assemblyModel.Collection.Add(testModel);
            }

            return(assemblyModel);
        }
示例#3
0
        private static IEnumerable <(PerformanceTestMessage performanceTestMessage, string metric, IEnumerable <double> values)> GetCollectedData(
            string assemblyFileName,
            CSVMetricReader reader,
            XUnitPerformanceMetricData xUnitPerformanceMetricData)
        {
            var testsFoundInAssembly = xUnitPerformanceMetricData.PerformanceTestMessages;

            foreach (var testFoundInAssembly in testsFoundInAssembly)
            {
                foreach (var(testCaseName, metric, values) in GetMeasurements(reader))
                {
                    if (values == null)
                    {
                        continue;
                    }

                    if (testCaseName == testFoundInAssembly.TestCase.DisplayName)
                    {
                        yield return(testFoundInAssembly, metric, values);
                    }
                }
            }
        }
示例#4
0
        /// <summary>
        ///     1. In the specified assembly, get the ETW providers set as assembly attributes (PerformanceTestInfo)
        ///     2. Check if the benchmark assembly request Precise Machine Counters(PMC) to be collected
        ///     3. Enable Kernel providers if needed
        ///     4. Get non-kernel ETW flags set and enable them
        ///     5. Run the benchmarks
        ///     6. Stop collecting ETW
        ///     7. Merge ETL files.
        /// </summary>
        /// <param name="xUnitPerformanceSessionData"></param>
        /// <param name="xUnitPerformanceMetricData"></param>
        /// <param name="action"></param>
        public static void Record(XUnitPerformanceSessionData xUnitPerformanceSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData, Action action)
        {
            const int bufferSizeMB  = 256;
            var       name          = $"{xUnitPerformanceSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitPerformanceSessionData.AssemblyFileName)}";
            var       etwOutputData = new ETWOutputData {
                KernelFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.kernel.etl"), // without this parameter, EnableKernelProvider will fail
                Name           = name,
                SessionName    = $"Performance-Api-Session-{xUnitPerformanceSessionData.RunId}",
                UserFileName   = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.etl"),
            };

            PrintProfilingInformation(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData);

            var kernelProviderInfo = xUnitPerformanceMetricData.Providers.OfType <KernelProviderInfo>().FirstOrDefault();
            var needKernelSession  = NeedSeparateKernelSession(kernelProviderInfo);

            if (needKernelSession && !CanEnableEnableKernelProvider)
            {
                const string message = "In order to capture kernel data the application is required to run as Administrator.";
                WriteErrorLine(message);
                throw new InvalidOperationException(message);
            }

            WriteDebugLine("> ETW capture start.");

            // Prior to Windows 8 (NT 6.2), all kernel events needed the special kernel session.
            using (var safeKernelSession = needKernelSession && CanEnableEnableKernelProvider ? MakeSafeTerminateTraceEventSession(KernelTraceEventParser.KernelSessionName, etwOutputData.KernelFileName) : null)
            {
                var kernelSession = safeKernelSession?.BaseDisposableObject;
                if (kernelSession != null)
                {
                    SetPreciseMachineCounters(xUnitPerformanceMetricData.Providers);
                    kernelSession.BufferSizeMB = bufferSizeMB;
                    var flags        = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords;
                    var stackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords;
                    kernelSession.EnableKernelProvider(flags, stackCapture);
                }

                using (var safeUserEventSession = MakeSafeTerminateTraceEventSession(etwOutputData.SessionName, etwOutputData.UserFileName))
                {
                    var userEventSession = safeUserEventSession.BaseDisposableObject;
                    userEventSession.BufferSizeMB = bufferSizeMB;

                    if (needKernelSession && CanEnableEnableKernelProvider)
                    {
                        userEventSession.EnableKernelProvider(KernelProvider.Default.Flags, KernelProvider.Default.StackCapture);
                    }

                    foreach (var provider in UserProvider.Defaults)
                    {
                        userEventSession.EnableProvider(provider.Guid, provider.Level, provider.Keywords);
                    }
                    foreach (var provider in xUnitPerformanceMetricData.Providers.OfType <UserProviderInfo>())
                    {
                        userEventSession.EnableProvider(provider.ProviderGuid, provider.Level, provider.Keywords);
                    }

                    action.Invoke();
                }
            }

            WriteDebugLine("> ETW capture stop.");

            // TODO: Decouple the code below.
            // Collect ETW profile data.
            //  TODO: Skip collecting kernel data if it was not captured! (data will be zero, there is no point to report it or upload it)
            WriteDebugLine("> ETW merge start.");
            TraceEventSession.MergeInPlace(etwOutputData.UserFileName, Console.Out);
            WriteDebugLine("> ETW merge stop.");
            xUnitPerformanceSessionData.CollectOutputFilesCallback(etwOutputData.UserFileName);

            var assemblyModel = GetAssemblyModel(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData.UserFileName, xUnitPerformanceSessionData.RunId, xUnitPerformanceMetricData.PerformanceTestMessages);
            var xmlFileName   = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.xml");

            new AssemblyModelCollection {
                assemblyModel
            }.Serialize(xmlFileName);
            xUnitPerformanceSessionData.CollectOutputFilesCallback(xmlFileName);

            var mdFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.md");
            var dt         = assemblyModel.GetStatistics();
            var mdTable    = MarkdownHelper.GenerateMarkdownTable(dt);

            MarkdownHelper.Write(mdFileName, mdTable);
            xUnitPerformanceSessionData.CollectOutputFilesCallback(mdFileName);

            Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));

            var csvFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.csv");

            dt.WriteToCSV(csvFileName);
            xUnitPerformanceSessionData.CollectOutputFilesCallback(csvFileName);
        }