public void SaveSummary(BenchmarkRunSummary summary)
{
using (var context = new BenchmarkContext(_contextOptions))
{
context.Runs.Add(new Run()
{
TestClassFullName = summary.TestClassFullName,
TestClass = summary.TestClass,
TestMethod = summary.TestMethod,
Variation = summary.Variation,
MachineName = summary.MachineName,
ProductReportingVersion = summary.ProductReportingVersion,
Framework = summary.Framework,
Architecture = summary.Architecture,
CustomData = summary.CustomData,
RunStarted = summary.RunStarted,
WarmupIterations = summary.WarmupIterations,
Iterations = summary.Iterations,
TimeElapsedAverage = summary.TimeElapsedAverage,
TimeElapsedPercentile99 = summary.TimeElapsedPercentile99,
TimeElapsedPercentile95 = summary.TimeElapsedPercentile95,
TimeElapsedPercentile90 = summary.TimeElapsedPercentile90,
MemoryDeltaAverage = summary.MemoryDeltaAverage,
MemoryDeltaPercentile99 = summary.MemoryDeltaPercentile99,
MemoryDeltaPercentile95 = summary.MemoryDeltaPercentile95,
MemoryDeltaPercentile90 = summary.MemoryDeltaPercentile90,
MemoryDeltaStandardDeviation = summary.MemoryDeltaStandardDeviation,
});
context.SaveChanges();
}
}
public static void SaveSummary(BenchmarkRunSummary summary)
{
Console.WriteLine(summary.ToString());
foreach (var connection in Connections.Value)
{
connection.SaveSummary(summary);
}
}
protected override async Task<RunSummary> RunTestAsync()
{
var runSummary = new BenchmarkRunSummary
{
TestClassFullName = TestCase.TestMethod.TestClass.Class.Name,
TestClass = TestCase.TestMethod.TestClass.Class.Name.Split('.').Last(),
TestMethod = TestCase.TestMethodName,
Variation = TestCase.Variation,
ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
RunStarted = DateTime.UtcNow,
MachineName = _machineName,
Framework = Microsoft.Extensions.Internal.RuntimeEnvironment.RuntimeType,
Architecture = IntPtr.Size > 4 ? "x64" : "x86",
WarmupIterations = TestCase.WarmupIterations,
Iterations = TestCase.Iterations,
CustomData = BenchmarkConfig.Instance.CustomData
};
for (var i = 0; i < TestCase.WarmupIterations; i++)
{
var runner = CreateRunner(i + 1, TestCase.WarmupIterations, TestCase.Variation, warmup: true);
runSummary.Aggregate(await runner.RunAsync());
}
for (var i = 0; i < TestCase.Iterations; i++)
{
TestCase.MetricCollector.Reset();
var runner = CreateRunner(i + 1, TestCase.Iterations, TestCase.Variation, warmup: false);
var iterationSummary = new BenchmarkIterationSummary();
iterationSummary.Aggregate(await runner.RunAsync(), TestCase.MetricCollector);
runSummary.Aggregate(iterationSummary);
}
if (runSummary.Failed != 0 || runSummary.Skipped != 0)
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"No valid results for {TestCase.DisplayName}. {runSummary.Failed} of {TestCase.Iterations + TestCase.WarmupIterations} iterations failed."));
}
else
{
runSummary.PopulateMetrics();
_diagnosticMessageSink.OnMessage(new DiagnosticMessage(runSummary.ToString()));
foreach (var database in BenchmarkConfig.Instance.ResultDatabases)
{
try
{
new SqlServerBenchmarkResultProcessor(database).SaveSummary(runSummary);
}
catch (Exception ex)
{
_diagnosticMessageSink.OnMessage(new DiagnosticMessage($"Failed to save results to {database}{Environment.NewLine} {ex}"));
throw;
}
}
}
return runSummary;
}
public void SaveSummary(BenchmarkRunSummary summary)
{
var serviceCollection = new ServiceCollection();
serviceCollection
.AddEntityFramework()
.AddSqlServer()
.AddDbContext <BenchmarkContext>(
options => options.UseSqlServer(_connectionString));
var serviceProvider = serviceCollection.BuildServiceProvider();
using (var context = serviceProvider.GetRequiredService <BenchmarkContext>())
{
context.Database.EnsureCreated();
context.Runs.Add(new Run()
{
TestClassFullName = summary.TestClassFullName,
TestClass = summary.TestClass,
TestMethod = summary.TestMethod,
Variation = summary.Variation,
MachineName = summary.MachineName,
ProductReportingVersion = summary.ProductReportingVersion,
Framework = summary.Framework,
Architecture = summary.Architecture,
CustomData = summary.CustomData,
RunStarted = summary.RunStarted,
WarmupIterations = summary.WarmupIterations,
Iterations = summary.Iterations,
TimeElapsedAverage = summary.TimeElapsedAverage,
TimeElapsedPercentile99 = summary.TimeElapsedPercentile99,
TimeElapsedPercentile95 = summary.TimeElapsedPercentile95,
TimeElapsedPercentile90 = summary.TimeElapsedPercentile90,
MemoryDeltaAverage = summary.MemoryDeltaAverage,
MemoryDeltaPercentile99 = summary.MemoryDeltaPercentile99,
MemoryDeltaPercentile95 = summary.MemoryDeltaPercentile95,
MemoryDeltaPercentile90 = summary.MemoryDeltaPercentile90,
MemoryDeltaStandardDeviation = summary.MemoryDeltaStandardDeviation,
});
context.SaveChanges();
}
}
public AntaresStartup()
{
_loggerFactory = new LoggerFactory();
_loggerFactory.AddConsole();
_summary = new BenchmarkRunSummary
{
TestClassFullName = GetType().FullName,
TestClass = GetType().Name,
RunStarted = DateTime.Now,
MachineName = GetMachineName(),
Iterations = _iterationCount,
Architecture = IntPtr.Size > 4 ? "x64" : "x86",
WarmupIterations = 0,
CustomData = BenchmarkConfig.Instance.CustomData,
ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
};
_rand = new Random((int)DateTime.Now.Ticks);
_username = GenerateRandomePassword();
_password = GenerateRandomePassword();
}
public static void Main(string[] args)
{
var logLocation = args[0];
var logFiles = Directory.GetFiles(logLocation, "log.xml", SearchOption.AllDirectories);
foreach (var file in logFiles)
{
var name = Path.GetFileName(Path.GetDirectoryName(file));
var parts = name.Split('.').ToArray();
Console.WriteLine($"Sample {parts[0]}, Framework {parts[1]}, Concurrency {parts[2]}.");
var summary = new BenchmarkRunSummary
{
TestClassFullName = $"Microsoft.AspNetCore.Tests.Throughput.BasicThroughputTests",
TestClass = $"BasicThroughputTests",
TestMethod = $"BasicTest",
ProductReportingVersion = BenchmarkConfig.Instance.ProductReportingVersion,
Iterations = 1,
WarmupIterations = 0,
Architecture = "x86",
CustomData = string.Empty,
Framework = parts[1]
};
using (var fs = File.OpenRead(file))
{
var xdoc = XDocument.Load(fs);
var info = xdoc.Descendants("section")
.First(elem => elem.Attribute("name").Value == "header")
.Descendants("data")
.ToDictionary(elem => elem.Attribute("name").Value, elem => elem.Value);
summary.RunStarted = DateTime.Parse(info["start"]);
summary.MachineName = info["host"];
summary.Variation = $"Sample={parts[0]} Concurrency={parts[2]}";
var data = xdoc.Descendants("section")
.First(elem => elem.Attribute("name").Value == "summary")
.Descendants("item").First()
.Descendants("data")
.ToDictionary(elem => elem.Attribute("name").Value, elem => elem.Value);
var rps = (long)double.Parse(data["rps"]);
summary.Aggregate(new BenchmarkIterationSummary {
TimeElapsed = rps
});
summary.PopulateMetrics();
try
{
BenchmarkResultProcessor.SaveSummary(summary);
}
catch (Exception ex)
{
Console.Error.WriteLine($"Failed to save results {Environment.NewLine} {ex}");
throw;
}
}
}
}
public void SaveSummary(BenchmarkRunSummary summary)
{
using (var conn = new SqlConnection(_connectionString))
{
conn.Open();
EnsureRunsTableCreated(conn);
WriteSummaryRecord(summary, conn);
}
}
private static void WriteSummaryRecord(BenchmarkRunSummary summary, SqlConnection conn)
{
var cmd = conn.CreateCommand();
cmd.CommandText = _insertCommand;
cmd.Parameters.AddWithValue("@TestClassFullName", summary.TestClassFullName);
cmd.Parameters.AddWithValue("@TestClass", summary.TestClass);
cmd.Parameters.AddWithValue("@TestMethod", summary.TestMethod);
cmd.Parameters.AddWithValue("@Variation", summary.Variation);
cmd.Parameters.AddWithValue("@MachineName", summary.MachineName);
cmd.Parameters.AddWithValue("@ProductReportingVersion", summary.ProductReportingVersion);
cmd.Parameters.AddWithValue("@Framework", summary.Framework);
cmd.Parameters.AddWithValue("@Architecture", summary.Architecture);
cmd.Parameters.Add("@CustomData", SqlDbType.NVarChar).Value = (object)summary.CustomData ?? DBNull.Value;
cmd.Parameters.AddWithValue("@RunStarted", summary.RunStarted);
cmd.Parameters.AddWithValue("@WarmupIterations", summary.WarmupIterations);
cmd.Parameters.AddWithValue("@Iterations", summary.Iterations);
cmd.Parameters.AddWithValue("@TimeElapsedAverage", summary.TimeElapsedAverage);
cmd.Parameters.AddWithValue("@TimeElapsedPercentile99", summary.TimeElapsedPercentile99);
cmd.Parameters.AddWithValue("@TimeElapsedPercentile95", summary.TimeElapsedPercentile95);
cmd.Parameters.AddWithValue("@TimeElapsedPercentile90", summary.TimeElapsedPercentile90);
cmd.Parameters.AddWithValue("@TimeElapsedStandardDeviation", summary.TimeElapsedStandardDeviation);
cmd.Parameters.AddWithValue("@MemoryDeltaAverage", summary.MemoryDeltaAverage);
cmd.Parameters.AddWithValue("@MemoryDeltaPercentile99", summary.MemoryDeltaPercentile99);
cmd.Parameters.AddWithValue("@MemoryDeltaPercentile95", summary.MemoryDeltaPercentile95);
cmd.Parameters.AddWithValue("@MemoryDeltaPercentile90", summary.MemoryDeltaPercentile90);
cmd.Parameters.AddWithValue("@MemoryDeltaStandardDeviation", summary.MemoryDeltaStandardDeviation);
cmd.ExecuteNonQuery();
}
