예제 #1
0
        private void ProcessResults(XUnitPerformanceSessionData xUnitSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData)
        {
            if (!File.Exists(Configuration.FileLogPath))
            {
                WriteWarningLine($"Results file '{Configuration.FileLogPath}' does not exist. Skipping processing of results.");
                return;
            }

            var reader = new CSVMetricReader(Configuration.FileLogPath);
            var fileNameWithoutExtension = $"{xUnitSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitSessionData.AssemblyFileName)}";

            var assemblyModel = AssemblyModel.Create(xUnitSessionData.AssemblyFileName, reader, xUnitPerformanceMetricData);
            var xmlFileName   = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.xml");

            new AssemblyModelCollection {
                assemblyModel
            }.Serialize(xmlFileName);
            xUnitSessionData.CollectOutputFilesCallback(xmlFileName);

            var dt         = assemblyModel.GetStatistics();
            var mdTable    = MarkdownHelper.GenerateMarkdownTable(dt);
            var mdFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.md");

            MarkdownHelper.Write(mdFileName, mdTable);
            xUnitSessionData.CollectOutputFilesCallback(mdFileName);
            Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));

            var csvFileName = Path.Combine(xUnitSessionData.OutputDirectory, $"{fileNameWithoutExtension}.csv");

            dt.WriteToCSV(csvFileName);
            xUnitSessionData.CollectOutputFilesCallback(csvFileName);

            BenchmarkEventSource.Log.Clear();
        }
예제 #2
0
        private void ProcessResults(string assemblyFileName, string sessionName, string outputDirectory, Action <string> collectOutputFilesCallback)
        {
            var reader = new CSVMetricReader(Configuration.FileLogPath);
            var fileNameWithoutExtension = $"{sessionName}-{Path.GetFileNameWithoutExtension(assemblyFileName)}";
            var statisticsFileName       = $"{fileNameWithoutExtension}-Statistics";
            var mdFileName = Path.Combine(outputDirectory, $"{statisticsFileName}.md");

            var assemblyModel = AssemblyModel.Create(assemblyFileName, reader);
            var xmlFileName   = Path.Combine(outputDirectory, $"{fileNameWithoutExtension}.xml");

            new AssemblyModelCollection {
                assemblyModel
            }.Serialize(xmlFileName);
            WriteInfoLine($"Performance results saved to \"{xmlFileName}\"");
            collectOutputFilesCallback(xmlFileName);

            var dt      = assemblyModel.GetStatistics();
            var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);

            MarkdownHelper.Write(mdFileName, mdTable);
            WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
            collectOutputFilesCallback(mdFileName);
            Console.WriteLine(mdTable);

            var csvFileName = Path.Combine(outputDirectory, $"{statisticsFileName}.csv");

            dt.WriteToCSV(csvFileName);
            WriteInfoLine($"Statistics written to \"{csvFileName}\"");
            collectOutputFilesCallback(csvFileName);

            BenchmarkEventSource.Log.Clear();
        }
예제 #3
0
        public void RunScenario(ProcessStartInfo processStartInfo, Action <ScenarioTestConfiguration> preIterationDel,
                                Action <ScenarioTestConfiguration> postIterationDel, Func <ScenarioTestConfiguration, ScenarioBenchmark> teardownDel)
        {
            int iterations = _scenarioTestConfiguration.Iterations;
            int timeout    = (int)(_scenarioTestConfiguration.TimeoutPerIteration.TotalMilliseconds);

            for (int i = 0; i < iterations; i++)
            {
                preIterationDel(_scenarioTestConfiguration);
                using (var p = new Process())
                {
                    p.StartInfo = processStartInfo;
                    p.Start();
                    if (p.WaitForExit(timeout) == false)
                    {
                        p.Kill();
                        Console.WriteLine("Timeouted!");
                        return;
                    }
                }
                postIterationDel(_scenarioTestConfiguration);
            }

            ScenarioBenchmark scenarioBenchmark = teardownDel(_scenarioTestConfiguration);

            if (scenarioBenchmark == null)
            {
                throw new InvalidOperationException("The Teardown Delegate should return a valid instance of ScenarioBenchmark.");
            }

            string scenarioNamespace = scenarioBenchmark.Namespace;

            scenarioBenchmark.Serialize(Configuration.RunId + "-" + scenarioNamespace + ".xml");

            var mdFileName  = Configuration.RunId + "-" + scenarioNamespace + "-Statistics.md";
            var csvFileName = Configuration.RunId + "-" + scenarioNamespace + "-Statistics.csv";

            var dt      = scenarioBenchmark.GetStatistics();
            var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);

            MarkdownHelper.Write(mdFileName, mdTable);
            WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
            Console.WriteLine(mdTable);

            dt.WriteToCSV(csvFileName);
            WriteInfoLine($"Statistics written to \"{csvFileName}\"");
        }
예제 #4
0
        /// <summary>
        /// Saves Markdown and CSV results for executed scenarios
        /// </summary>
        /// <param name="scenarios">The scenarios to save results for</param>
        /// <param name="fileNameWithoutExtension">The filename (without extension) to use to save results</param>
        /// <param name="includeScenarioNameColumn">Indicates whether scenario name should be included in the tables as the first column</param>
        public void WriteTableResults(IEnumerable <ScenarioBenchmark> scenarios, string fileNameWithoutExtension, bool includeScenarioNameColumn)
        {
            var dt = ScenarioBenchmark.GetEmptyTable(includeScenarioNameColumn ? null : scenarios.First().Name);

            foreach (var scenario in scenarios)
            {
                scenario.AddRowsToTable(dt, scenario.GetStatistics(), includeScenarioNameColumn);
            }

            var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);

            var csvFileName = $"{fileNameWithoutExtension}.csv";

            dt.WriteToCSV(csvFileName);
            LogFileSaved(csvFileName);

            var mdFileName = $"{fileNameWithoutExtension}.md";

            MarkdownHelper.Write(mdFileName, mdTable);
            LogFileSaved(mdFileName);
            Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));
        }
예제 #5
0
        /// <summary>
        ///     1. In the specified assembly, get the ETW providers set as assembly attributes (PerformanceTestInfo)
        ///     2. Check if the benchmark assembly request Precise Machine Counters(PMC) to be collected
        ///     3. Enable Kernel providers if needed
        ///     4. Get non-kernel ETW flags set and enable them
        ///     5. Run the benchmarks
        ///     6. Stop collecting ETW
        ///     7. Merge ETL files.
        /// </summary>
        /// <param name="xUnitPerformanceSessionData"></param>
        /// <param name="xUnitPerformanceMetricData"></param>
        /// <param name="action"></param>
        public static void Record(XUnitPerformanceSessionData xUnitPerformanceSessionData, XUnitPerformanceMetricData xUnitPerformanceMetricData, Action action)
        {
            const int bufferSizeMB  = 256;
            var       name          = $"{xUnitPerformanceSessionData.RunId}-{Path.GetFileNameWithoutExtension(xUnitPerformanceSessionData.AssemblyFileName)}";
            var       etwOutputData = new ETWOutputData {
                KernelFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.kernel.etl"), // without this parameter, EnableKernelProvider will fail
                Name           = name,
                SessionName    = $"Performance-Api-Session-{xUnitPerformanceSessionData.RunId}",
                UserFileName   = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{name}.etl"),
            };

            PrintProfilingInformation(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData);

            var kernelProviderInfo = xUnitPerformanceMetricData.Providers.OfType <KernelProviderInfo>().FirstOrDefault();
            var needKernelSession  = NeedSeparateKernelSession(kernelProviderInfo);

            if (needKernelSession && !CanEnableEnableKernelProvider)
            {
                const string message = "In order to capture kernel data the application is required to run as Administrator.";
                WriteErrorLine(message);
                throw new InvalidOperationException(message);
            }

            WriteDebugLine("> ETW capture start.");

            // Prior to Windows 8 (NT 6.2), all kernel events needed the special kernel session.
            using (var safeKernelSession = needKernelSession && CanEnableEnableKernelProvider ? MakeSafeTerminateTraceEventSession(KernelTraceEventParser.KernelSessionName, etwOutputData.KernelFileName) : null)
            {
                var kernelSession = safeKernelSession?.BaseDisposableObject;
                if (kernelSession != null)
                {
                    SetPreciseMachineCounters(xUnitPerformanceMetricData.Providers);
                    kernelSession.BufferSizeMB = bufferSizeMB;
                    var flags        = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords;
                    var stackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords;
                    kernelSession.EnableKernelProvider(flags, stackCapture);
                }

                using (var safeUserEventSession = MakeSafeTerminateTraceEventSession(etwOutputData.SessionName, etwOutputData.UserFileName))
                {
                    var userEventSession = safeUserEventSession.BaseDisposableObject;
                    userEventSession.BufferSizeMB = bufferSizeMB;

                    if (needKernelSession && CanEnableEnableKernelProvider)
                    {
                        userEventSession.EnableKernelProvider(KernelProvider.Default.Flags, KernelProvider.Default.StackCapture);
                    }

                    foreach (var provider in UserProvider.Defaults)
                    {
                        userEventSession.EnableProvider(provider.Guid, provider.Level, provider.Keywords);
                    }
                    foreach (var provider in xUnitPerformanceMetricData.Providers.OfType <UserProviderInfo>())
                    {
                        userEventSession.EnableProvider(provider.ProviderGuid, provider.Level, provider.Keywords);
                    }

                    action.Invoke();
                }
            }

            WriteDebugLine("> ETW capture stop.");

            // TODO: Decouple the code below.
            // Collect ETW profile data.
            //  TODO: Skip collecting kernel data if it was not captured! (data will be zero, there is no point to report it or upload it)
            WriteDebugLine("> ETW merge start.");
            TraceEventSession.MergeInPlace(etwOutputData.UserFileName, Console.Out);
            WriteDebugLine("> ETW merge stop.");
            xUnitPerformanceSessionData.CollectOutputFilesCallback(etwOutputData.UserFileName);

            var assemblyModel = GetAssemblyModel(xUnitPerformanceSessionData.AssemblyFileName, etwOutputData.UserFileName, xUnitPerformanceSessionData.RunId, xUnitPerformanceMetricData.PerformanceTestMessages);
            var xmlFileName   = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.xml");

            new AssemblyModelCollection {
                assemblyModel
            }.Serialize(xmlFileName);
            xUnitPerformanceSessionData.CollectOutputFilesCallback(xmlFileName);

            var mdFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.md");
            var dt         = assemblyModel.GetStatistics();
            var mdTable    = MarkdownHelper.GenerateMarkdownTable(dt);

            MarkdownHelper.Write(mdFileName, mdTable);
            xUnitPerformanceSessionData.CollectOutputFilesCallback(mdFileName);

            Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));

            var csvFileName = Path.Combine(xUnitPerformanceSessionData.OutputDirectory, $"{etwOutputData.Name}.csv");

            dt.WriteToCSV(csvFileName);
            xUnitPerformanceSessionData.CollectOutputFilesCallback(csvFileName);
        }
        /// <summary>
        /// Executes the benchmark scenario specified by the parameter
        /// containing the process start information.<br/>
        /// The process component will wait, for the benchmark scenario to exit,
        /// the time specified on the configuration argument.<br/>
        /// If the benchmark scenario has not exited, then it will immediately
        /// stop the associated process, and a TimeoutException will be thrown.
        /// </summary>
        /// <param name="processStartInfo">The ProcessStartInfo that contains the information that is used to start the benchmark scenario process.</param>
        /// <param name="preIterationDelegate">The action that will be executed before every benchmark scenario execution.</param>
        /// <param name="postIterationDelegate">The action that will be executed after every benchmark scenario execution.</param>
        /// <param name="teardownDelegate">The action that will be executed after running all benchmark scenario iterations.</param>
        /// <param name="scenarioConfiguration">ScenarioConfiguration object that defined the scenario execution.</param>
        public void RunScenario(
            ProcessStartInfo processStartInfo,
            Action preIterationDelegate,
            Action postIterationDelegate,
            Func <ScenarioBenchmark> teardownDelegate,
            ScenarioConfiguration scenarioConfiguration)
        {
            if (processStartInfo == null)
            {
                throw new ArgumentNullException($"{nameof(processStartInfo)} cannot be null.");
            }
            if (teardownDelegate == null)
            {
                throw new ArgumentNullException($"{nameof(teardownDelegate)} cannot be null.");
            }
            if (scenarioConfiguration == null)
            {
                throw new ArgumentNullException($"{nameof(scenarioConfiguration)} cannot be null.");
            }

            // Make a copy of the user input to avoid potential bugs due to changes behind the scenes.
            var configuration = new ScenarioConfiguration(scenarioConfiguration);

            for (int i = 0; i < configuration.Iterations; ++i)
            {
                preIterationDelegate?.Invoke();

                // TODO: Start scenario profiling.

                using (var process = new Process())
                {
                    process.StartInfo = processStartInfo;
                    process.Start();
                    if (process.WaitForExit((int)(configuration.TimeoutPerIteration.TotalMilliseconds)) == false)
                    {
                        process.Kill();
                        throw new TimeoutException("Running benchmark scenario has timed out.");
                    }

                    // Check for the exit code.
                    if (!configuration.SuccessExitCodes.Contains(process.ExitCode))
                    {
                        throw new Exception($"'{processStartInfo.FileName}' exited with an invalid exit code: {process.ExitCode}");
                    }
                }

                // TODO: Stop scenario profiling.

                postIterationDelegate?.Invoke();
            }

            ScenarioBenchmark scenarioBenchmark = teardownDelegate();

            if (scenarioBenchmark == null)
            {
                throw new InvalidOperationException("The Teardown Delegate should return a valid instance of ScenarioBenchmark.");
            }
            scenarioBenchmark.Serialize(Configuration.RunId + "-" + scenarioBenchmark.Namespace + ".xml");

            var dt      = scenarioBenchmark.GetStatistics();
            var mdTable = MarkdownHelper.GenerateMarkdownTable(dt);

            var mdFileName = $"{Configuration.RunId}-{scenarioBenchmark.Namespace}-Statistics.md";

            MarkdownHelper.Write(mdFileName, mdTable);
            WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
            Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));

            var csvFileName = $"{Configuration.RunId}-{scenarioBenchmark.Namespace}-Statistics.csv";

            dt.WriteToCSV(csvFileName);
            WriteInfoLine($"Statistics written to \"{csvFileName}\"");
        }
예제 #7
0
        /// <summary>
        ///     1. In the specified assembly, get the ETW providers set as assembly attributes (PerformanceTestInfo)
        ///     2. Check if the benchmark assembly request Precise Machine Counters(PMC) to be collected
        ///     3. Enable Kernel providers if needed
        ///     4. Get non-kernel ETW flags set and enable them
        ///     5. Run the benchmarks
        ///     6. Stop collecting ETW
        ///     7. Merge ETL files.
        /// </summary>
        /// <param name="assemblyFileName"></param>
        /// <param name="runId"></param>
        /// <param name="outputDirectory"></param>
        /// <param name="action"></param>
        /// <param name="collectOutputFilesCallback">Callback used to collect a list of files generated.</param>
        /// <returns></returns>
        public static void Record(string assemblyFileName, string runId, string outputDirectory, Action action, Action <string> collectOutputFilesCallback)
        {
            if (TraceEventSession.IsElevated() != true)
            {
                const string errMessage = "In order to profile, application is required to run as Administrator.";
                WriteErrorLine(errMessage);
                throw new InvalidOperationException(errMessage);
            }

            const int bufferSizeMB       = 256;
            var       sessionName        = $"Performance-Api-Session-{runId}";
            var       name               = $"{runId}-{Path.GetFileNameWithoutExtension(assemblyFileName)}";
            var       userFullFileName   = Path.Combine(outputDirectory, $"{name}.etl");
            var       kernelFullFileName = Path.Combine(outputDirectory, $"{name}.kernel.etl"); // without this parameter, EnableKernelProvider will fail

            PrintProfilingInformation(assemblyFileName, sessionName, userFullFileName);

            (var providers, var performanceTestMessages) = XunitBenchmark.GetMetadata(assemblyFileName);
            var kernelProviderInfo = providers.OfType <KernelProviderInfo>().FirstOrDefault();

            var needKernelSession = NeedSeparateKernelSession(kernelProviderInfo);

            using (var safeKernelSession = needKernelSession ? MakeSafeTerminateTraceEventSession(KernelTraceEventParser.KernelSessionName, kernelFullFileName) : null)
            {
                var kernelSession = safeKernelSession?.BaseDisposableObject;
                if (kernelSession != null)
                {
                    SetPreciseMachineCounters(providers);
                    kernelSession.BufferSizeMB = bufferSizeMB;
                    var flags        = (KernelTraceEventParser.Keywords)kernelProviderInfo.Keywords;
                    var stackCapture = (KernelTraceEventParser.Keywords)kernelProviderInfo.StackKeywords;
                    kernelSession.EnableKernelProvider(flags, stackCapture);
                }

                using (var safeUserEventSession = MakeSafeTerminateTraceEventSession(sessionName, userFullFileName))
                {
                    var userEventSession = safeUserEventSession.BaseDisposableObject;
                    userEventSession.BufferSizeMB = bufferSizeMB;

                    var flags        = KernelTraceEventParser.Keywords.Process | KernelTraceEventParser.Keywords.ImageLoad | KernelTraceEventParser.Keywords.Thread;
                    var stackCapture = KernelTraceEventParser.Keywords.Profile | KernelTraceEventParser.Keywords.ContextSwitch;
                    userEventSession.EnableKernelProvider(flags, stackCapture);

                    foreach (var userProviderInfo in providers.OfType <UserProviderInfo>())
                    {
                        userEventSession.EnableProvider(userProviderInfo.ProviderGuid, userProviderInfo.Level, userProviderInfo.Keywords);
                    }

                    action.Invoke();
                }
            }

            TraceEventSession.MergeInPlace(userFullFileName, Console.Out);
            WriteInfoLine($"ETW Tracing Session saved to \"{userFullFileName}\"");
            collectOutputFilesCallback(userFullFileName);

            var assemblyModel = GetAssemblyModel(assemblyFileName, userFullFileName, runId, performanceTestMessages);
            var xmlFileName   = Path.Combine(outputDirectory, $"{name}.xml");

            new AssemblyModelCollection {
                assemblyModel
            }.Serialize(xmlFileName);
            WriteInfoLine($"Performance results saved to \"{xmlFileName}\"");
            collectOutputFilesCallback(xmlFileName);

            var mdFileName = Path.Combine(outputDirectory, $"{name}.md");
            var dt         = assemblyModel.GetStatistics();
            var mdTable    = MarkdownHelper.GenerateMarkdownTable(dt);

            MarkdownHelper.Write(mdFileName, mdTable);
            WriteInfoLine($"Markdown file saved to \"{mdFileName}\"");
            collectOutputFilesCallback(mdFileName);
            Console.WriteLine(MarkdownHelper.ToTrimmedTable(mdTable));

            var csvFileName = Path.Combine(outputDirectory, $"{name}.csv");

            dt.WriteToCSV(csvFileName);
            WriteInfoLine($"Statistics written to \"{csvFileName}\"");
            collectOutputFilesCallback(csvFileName);
        }