protected override async Task <RunSummary> RunTestAsync()
{
var runSummary = new BenchmarkRunSummary
{
TestClassFullName = TestCase.TestMethod.TestClass.Class.Name,
TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
TestMethod = TestCase.TestMethodName,
Variation = TestCase.Variation,
ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
RunStarted = DateTime.UtcNow,
MachineName = _machineName,
Framework = TestCase.Framework ?? _framework,
Architecture = IntPtr.Size > 4 ? "x64" : "x86",
WarmupIterations = TestCase.WarmupIterations,
Iterations = TestCase.Iterations,
CustomData = BenchmarkConfig.Instance.CustomData
};
for (var i = 0; i < TestCase.WarmupIterations; i++)
{
var runner = CreateRunner(i + 1, TestCase, warmup: true);
runSummary.Aggregate(await runner.RunAsync());
}
for (var i = 0; i < TestCase.Iterations; i++)
{
TestCase.MetricCollector.Reset();
var runner = CreateRunner(i + 1, TestCase, warmup: false);
var iterationSummary = new BenchmarkIterationSummary();
iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
runSummary.Aggregate(iterationSummary);
}
if (runSummary.Failed != 0)
{
_diagnosticMessageSink.OnMessage(
new XunitDiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed."));
}
else
{
runSummary.PopulateMetrics();
_diagnosticMessageSink.OnMessage(new XunitDiagnosticMessage(runSummary.ToString()));
try
{
BenchmarkResultProcessor.SaveSummary(runSummary);
}
catch (Exception ex)
{
_diagnosticMessageSink.OnMessage(
new XunitDiagnosticMessage($"Failed to save results {Environment.NewLine} {ex}"));
throw;
}
}
return(runSummary);
}
protected void SaveSummary(ILogger logger)
{
try
{
BenchmarkResultProcessor.SaveSummary(_summary);
}
catch (Exception ex)
{
logger.LogError($"Failed to save results {Environment.NewLine} {ex}");
throw;
}
}
public static void Main(string[] args)
{
var logLocation = args[0];
var logFiles = Directory.GetFiles(logLocation, "log.xml", SearchOption.AllDirectories);
foreach (var file in logFiles)
{
var name = Path.GetFileName(Path.GetDirectoryName(file));
var parts = name.Split('.').ToArray();
Console.WriteLine($"Sample {parts[0]}, Framework {parts[1]}, Concurrency {parts[2]}.");
var summary = new BenchmarkRunSummary
{
TestClassFullName = $"Microsoft.AspNetCore.Tests.Throughput.BasicThroughputTests",
TestClass = $"BasicThroughputTests",
TestMethod = $"BasicTest",
ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
Iterations = 1,
WarmupIterations = 0,
Architecture = "x86",
CustomData = string.Empty,
Framework = parts[1]
};
using (var fs = File.OpenRead(file))
{
var xdoc = XDocument.Load(fs);
var info = xdoc.Descendants("section")
.First(elem => elem.Attribute("name").Value == "header")
.Descendants("data")
.ToDictionary(elem => elem.Attribute("name").Value, elem => elem.Value);
summary.RunStarted = DateTime.Parse(info["start"]);
summary.MachineName = info["host"];
summary.Variation = $"Sample={parts[0]} Concurrency={parts[2]}";
var data = xdoc.Descendants("section")
.First(elem => elem.Attribute("name").Value == "summary")
.Descendants("item").First()
.Descendants("data")
.ToDictionary(elem => elem.Attribute("name").Value, elem => elem.Value);
var rps = (long)double.Parse(data["rps"]);
summary.Aggregate(new BenchmarkIterationSummary {
TimeElapsed = rps
});
summary.PopulateMetrics();
try
{
BenchmarkResultProcessor.SaveSummary(summary);
}
catch (Exception ex)
{
Console.Error.WriteLine($"Failed to save results {Environment.NewLine} {ex}");
throw;
}
}
}
}
