public static List <Test> GenerateTestsFromXml(StreamReader stream) { XmlSerializer serializer = new XmlSerializer(typeof(ScenarioBenchmark)); ScenarioBenchmark scenarioBenchmark = (ScenarioBenchmark)serializer.Deserialize(stream); Console.WriteLine(scenarioBenchmark.Name); List <Test> tests = new List <Test>(); foreach (ScenarioBenchmarkTest scenarioBenchmarkTest in scenarioBenchmark.Tests) { Test test = new Test(); test.Categories.Add("DotnetCoreSdk"); test.Name = scenarioBenchmark.Name + "." + scenarioBenchmarkTest.Name; test.Counters.Add(new Counter { Name = scenarioBenchmarkTest.Performance.metrics.ExecutionTime.displayName, TopCounter = true, DefaultCounter = true, HigherIsBetter = false, MetricName = scenarioBenchmarkTest.Performance.metrics.ExecutionTime.unit, Results = scenarioBenchmarkTest .Performance .iterations .Select(i => decimal.ToDouble(i.ExecutionTime)) .ToList() }); tests.Add(test); } return(tests); }
static ScenarioBenchmark GetScenarioBenchmark(string name) { ScenarioBenchmark scenario; if (_scenarios.TryGetValue(name, out scenario)) { return(scenario); } scenario = new ScenarioBenchmark(name); _scenarios[name] = scenario; return(scenario); }
private static void PostRun(ScenarioBenchmark scenario) { // The XUnit output doesn't print the benchmark name, so print it now. Console.WriteLine("{0}", CurrentBenchmark.Name); CurrentBenchmark.Compute(); addMeasurement(ref scenario, "MSIL Unlinked", SizeMetric, CurrentBenchmark.UnlinkedMsilSize); addMeasurement(ref scenario, "MSIL Linked", SizeMetric, CurrentBenchmark.LinkedMsilSize); addMeasurement(ref scenario, "MSIL Reduction", PercMetric, CurrentBenchmark.MsilSizeReduction); addMeasurement(ref scenario, "Total Uninked", SizeMetric, CurrentBenchmark.UnlinkedDirSize); addMeasurement(ref scenario, "Total Linked", SizeMetric, CurrentBenchmark.LinkedDirSize); addMeasurement(ref scenario, "Total Reduction", PercMetric, CurrentBenchmark.DirSizeReduction); }
private static ScenarioBenchmark PostProcessing() { PrintHeader("Starting POST"); var scenarioBenchmark = new ScenarioBenchmark("MusicStore") { Namespace = "JitBench" }; // Create (measured) test entries for this scenario. var startup = new ScenarioTestModel("Startup"); scenarioBenchmark.Tests.Add(startup); var request = new ScenarioTestModel("First Request"); scenarioBenchmark.Tests.Add(request); // TODO: add response time once jit bench is updated to // report more reasonable numbers. // Add measured metrics to each test. startup.Performance.Metrics.Add(new MetricModel { Name = "Duration", DisplayName = "Duration", Unit = "ms" }); request.Performance.Metrics.Add(new MetricModel { Name = "Duration", DisplayName = "Duration", Unit = "ms" }); for (int i = 0; i < s_iterations; ++i) { var startupIteration = new IterationModel { Iteration = new Dictionary <string, double>() }; startupIteration.Iteration.Add("Duration", s_startupTimes[i]); startup.Performance.IterationModels.Add(startupIteration); var requestIteration = new IterationModel { Iteration = new Dictionary <string, double>() }; requestIteration.Iteration.Add("Duration", s_requestTimes[i]); request.Performance.IterationModels.Add(requestIteration); } return(scenarioBenchmark); }
private static void addMeasurement(ref ScenarioBenchmark scenario, string name, MetricModel metric, double value) { var iteration = new IterationModel { Iteration = new Dictionary <string, double>() }; iteration.Iteration.Add(metric.Name, value); var size = new ScenarioTestModel(name); size.Performance.Metrics.Add(metric); size.Performance.IterationModels.Add(iteration); scenario.Tests.Add(size); }
private static ScenarioBenchmark PostRun() { // The XUnit output doesn't print the benchmark name, so print it now. Console.WriteLine("{0}", CurrentBenchmark.Name); var scenario = new ScenarioBenchmark(CurrentBenchmark.Name) { Namespace = "LinkBench" }; CurrentBenchmark.Compute(); addMeasurement(ref scenario, "MSIL Unlinked", SizeMetric, CurrentBenchmark.UnlinkedMsilSize); addMeasurement(ref scenario, "MSIL Linked", SizeMetric, CurrentBenchmark.LinkedMsilSize); addMeasurement(ref scenario, "MSIL Reduction", PercMetric, CurrentBenchmark.MsilSizeReduction); addMeasurement(ref scenario, "Total Uninked", SizeMetric, CurrentBenchmark.UnlinkedDirSize); addMeasurement(ref scenario, "Total Linked", SizeMetric, CurrentBenchmark.LinkedDirSize); addMeasurement(ref scenario, "Total Reduction", PercMetric, CurrentBenchmark.DirSizeReduction); return(scenario); }
private static ScenarioBenchmark AddEtwData( ScenarioBenchmark scenarioBenchmark, ScenarioExecutionResult scenarioExecutionResult, IReadOnlyCollection <string> processesOfInterest, IReadOnlyCollection <string> modulesOfInterest) { var metricModels = scenarioExecutionResult.PerformanceMonitorCounters .Select(pmc => new MetricModel { DisplayName = pmc.DisplayName, Name = pmc.Name, Unit = pmc.Unit, }); // Get the list of processes of interest. Console.WriteLine($"Parsing: {scenarioExecutionResult.EventLogFileName}"); var processes = new SimpleTraceEventParser().GetProfileData(scenarioExecutionResult); // Extract the Pmc data for each one of the processes. foreach (var process in processes) { if (!processesOfInterest.Any(p => p.Equals(process.Name, StringComparison.OrdinalIgnoreCase))) { continue; } var processTest = scenarioBenchmark.Tests .SingleOrDefault(t => t.Name == process.Name && t.Namespace == ""); if (processTest == null) { processTest = new ScenarioTestModel(process.Name) { Namespace = "", }; scenarioBenchmark.Tests.Add(processTest); // Add metrics definitions. processTest.Performance.Metrics.Add(ElapsedTimeMilliseconds); processTest.Performance.Metrics.AddRange(metricModels); } var processIterationModel = new IterationModel { Iteration = new Dictionary <string, double>() }; processTest.Performance.IterationModels.Add(processIterationModel); processIterationModel.Iteration.Add( ElapsedTimeMilliseconds.Name, process.LifeSpan.Duration.TotalMilliseconds); // Add process metrics values. foreach (var pmcData in process.PerformanceMonitorCounterData) { processIterationModel.Iteration.Add(pmcData.Key.Name, pmcData.Value); } foreach (var module in process.Modules) { var moduleName = Path.GetFileName(module.FullName); if (modulesOfInterest.Any(m => m.Equals(moduleName, StringComparison.OrdinalIgnoreCase))) { var moduleTestName = $"{moduleName}"; var moduleTest = scenarioBenchmark.Tests .SingleOrDefault(t => t.Name == moduleTestName && t.Namespace == process.Name); if (moduleTest == null) { moduleTest = new ScenarioTestModel(moduleTestName) { Namespace = process.Name, Separator = "!", }; scenarioBenchmark.Tests.Add(moduleTest); // Add metrics definitions. moduleTest.Performance.Metrics.AddRange(metricModels); } var moduleIterationModel = new IterationModel { Iteration = new Dictionary <string, double>() }; moduleTest.Performance.IterationModels.Add(moduleIterationModel); // 5. Add module metrics values. foreach (var pmcData in module.PerformanceMonitorCounterData) { moduleIterationModel.Iteration.Add(pmcData.Key.Name, pmcData.Value); } } } } return(scenarioBenchmark); }
private ScenarioBenchmark PostRun( string scenarioTestModelName, IReadOnlyCollection <string> processesOfInterest, IReadOnlyCollection <string> modulesOfInterest) { PrintHeader("Post-Processing scenario data."); var scenarioBenchmark = new ScenarioBenchmark(_scenarioBenchmarkName); foreach (var iter in IterationsData) { var scenarioExecutionResult = iter.ScenarioExecutionResult; var scenarioTestModel = scenarioBenchmark.Tests .SingleOrDefault(t => t.Name == scenarioTestModelName); if (scenarioTestModel == null) { scenarioTestModel = new ScenarioTestModel(scenarioTestModelName); scenarioBenchmark.Tests.Add(scenarioTestModel); // Add measured metrics to each test. scenarioTestModel.Performance.Metrics.Add(ElapsedTimeMilliseconds); } scenarioTestModel.Performance.IterationModels.Add(new IterationModel { Iteration = new Dictionary <string, double> { { ElapsedTimeMilliseconds.Name, (scenarioExecutionResult.ProcessExitInfo.ExitTime - scenarioExecutionResult.ProcessExitInfo.StartTime).TotalMilliseconds }, } }); // Create (measured) test entries for this scenario. var startup = scenarioBenchmark.Tests .SingleOrDefault(t => t.Name == "Startup" && t.Namespace == scenarioTestModel.Name); if (startup == null) { startup = new ScenarioTestModel("Startup") { Namespace = scenarioTestModel.Name, }; scenarioBenchmark.Tests.Add(startup); // Add measured metrics to each test. startup.Performance.Metrics.Add(ElapsedTimeMilliseconds); } var firstRequest = scenarioBenchmark.Tests .SingleOrDefault(t => t.Name == "First Request" && t.Namespace == scenarioTestModel.Name); if (firstRequest == null) { firstRequest = new ScenarioTestModel("First Request") { Namespace = scenarioTestModel.Name, }; scenarioBenchmark.Tests.Add(firstRequest); // Add measured metrics to each test. firstRequest.Performance.Metrics.Add(ElapsedTimeMilliseconds); } startup.Performance.IterationModels.Add(new IterationModel { Iteration = new Dictionary <string, double> { { ElapsedTimeMilliseconds.Name, iter.StartupTime }, }, }); firstRequest.Performance.IterationModels.Add(new IterationModel { Iteration = new Dictionary <string, double> { { ElapsedTimeMilliseconds.Name, iter.FirstRequestTime }, }, }); if (!string.IsNullOrWhiteSpace(iter.ScenarioExecutionResult.EventLogFileName) && File.Exists(iter.ScenarioExecutionResult.EventLogFileName)) { // Adding ETW data. scenarioBenchmark = AddEtwData( scenarioBenchmark, iter.ScenarioExecutionResult, processesOfInterest, modulesOfInterest); } } for (int i = scenarioBenchmark.Tests.Count - 1; i >= 0; i--) { if (scenarioBenchmark.Tests[i].Performance.IterationModels.All(iter => iter.Iteration.Count == 0)) { scenarioBenchmark.Tests.RemoveAt(i); } } return(scenarioBenchmark); }
private static string GetFilePathWithoutExtension(string outputDir, string runId, ScenarioBenchmark benchmark) { return(Path.Combine(outputDir, $"{runId}-{benchmark.Name}")); }
public static void ConvertRunResult(ScenarioBenchmark scenario, BenchmarkRunResult runResult) { scenario.Tests = new List <ScenarioTestModel>(); scenario.Tests.AddRange(ConvertRunResult(runResult)); }