protected override async Task <RunSummary> RunTestAsync() { var runSummary = new BenchmarkRunSummary { TestClassFullName = TestCase.TestMethod.TestClass.Class.Name, TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(), TestMethod = TestCase.TestMethodName, Variation = TestCase.Variation, ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion, RunStarted = DateTime.UtcNow, MachineName = _machineName, Framework = TestCase.Framework ?? _framework, Architecture = IntPtr.Size > 4 ? "x64" : "x86", WarmupIterations = TestCase.WarmupIterations, Iterations = TestCase.Iterations, CustomData = BenchmarkConfig.Instance.CustomData }; for (var i = 0; i < TestCase.WarmupIterations; i++) { var runner = CreateRunner(i + 1, TestCase, warmup: true); runSummary.Aggregate(await runner.RunAsync()); } for (var i = 0; i < TestCase.Iterations; i++) { TestCase.MetricCollector.Reset(); var runner = CreateRunner(i + 1, TestCase, warmup: false); var iterationSummary = new BenchmarkIterationSummary(); iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector); runSummary.Aggregate(iterationSummary); } if (runSummary.Failed != 0) { _diagnosticMessageSink.OnMessage( new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed.")); } else { runSummary.PopulateMetrics(); _diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString())); try { BenchmarkResultProcessor.SaveSummary(runSummary); } catch (Exception ex) { _diagnosticMessageSink.OnMessage( new XunitDiagnosticMessage($"Failed to save results {Environment.NewLine} {ex}")); throw; } } return(runSummary); }
protected void SaveSummary(ILogger logger) { try { BenchmarkResultProcessor.SaveSummary(_summary); } catch (Exception ex) { logger.LogError($"Failed to save results {Environment.NewLine} {ex}"); throw; } }
public static void Main(string[] args) { var logLocation = args[0]; var logFiles = Directory.GetFiles(logLocation, "log.xml", SearchOption.AllDirectories); foreach (var file in logFiles) { var name = Path.GetFileName(Path.GetDirectoryName(file)); var parts = name.Split('.').ToArray(); Console.WriteLine($"Sample {parts[0]}, Framework {parts[1]}, Concurrency {parts[2]}."); var summary = new BenchmarkRunSummary { TestClassFullName = $"Microsoft.AspNetCore.Tests.Throughput.BasicThroughputTests", TestClass = $"BasicThroughputTests", TestMethod = $"BasicTest", ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion, Iterations = 1, WarmupIterations = 0, Architecture = "x86", CustomData = string.Empty, Framework = parts[1] }; using (var fs = File.OpenRead(file)) { var xdoc = XDocument.Load(fs); var info = xdoc.Descendants("section") .First(elem => elem.Attribute("name").Value == "header") .Descendants("data") .ToDictionary(elem => elem.Attribute("name").Value, elem => elem.Value); summary.RunStarted = DateTime.Parse(info["start"]); summary.MachineName = info["host"]; summary.Variation = $"Sample={parts[0]} Concurrency={parts[2]}"; var data = xdoc.Descendants("section") .First(elem => elem.Attribute("name").Value == "summary") .Descendants("item").First() .Descendants("data") .ToDictionary(elem => elem.Attribute("name").Value, elem => elem.Value); var rps = (long)double.Parse(data["rps"]); summary.Aggregate(new BenchmarkIterationSummary { TimeElapsed = rps }); summary.PopulateMetrics(); try { BenchmarkResultProcessor.SaveSummary(summary); } catch (Exception ex) { Console.Error.WriteLine($"Failed to save results {Environment.NewLine} {ex}"); throw; } } } }