public static IEnumerable <VerifierBenchmark> EnumerateBenchmarks()
{
foreach (var config in BenchmarkConfig.GenerateConfigurations())
{
yield return(BenchmarkBuilder.Build(config, DatabasePath));
}
}
public void Should_build_when_at_least_one_metric_assigned()
{
var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.Gen2, AssertionType.Total,
Assertion.Empty);
var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
new List <IBenchmarkSetting>()
{
gcBenchmark, memoryBenchmark, counterBenchmark
},
new Dictionary <MetricName, MetricsCollectorSelector>()
{
{ gcBenchmark.MetricName, new GcCollectionsSelector() },
{ counterBenchmark.MetricName, new CounterSelector() },
{ memoryBenchmark.MetricName, new TotalMemorySelector() }
});
var builder = new BenchmarkBuilder(settings);
var run = builder.NewRun(WarmupData.PreWarmup);
Assert.Equal(3, run.MeasureCount);
Assert.Equal(1, run.Counters.Count);
Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
}
public void Should_build_when_exactly_one_metric_assigned()
{
var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
new GcBenchmarkSetting[0], new MemoryBenchmarkSetting[0], new CounterBenchmarkSetting[] { counterBenchmark });
var builder = new BenchmarkBuilder(settings);
var run = builder.NewRun(WarmupData.PreWarmup);
Assert.Equal(1, run.MeasureCount);
Assert.Equal(1, run.Counters.Count);
Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
}
public void Should_build_when_at_least_one_metric_assigned()
{
var counterBenchmark = new CounterBenchmarkSetting("Test", AssertionType.Total, Assertion.Empty);
var gcBenchmark = new GcBenchmarkSetting(GcMetric.TotalCollections, GcGeneration.AllGc, AssertionType.Total,
Assertion.Empty);
var memoryBenchmark = new MemoryBenchmarkSetting(MemoryMetric.TotalBytesAllocated, Assertion.Empty);
var settings = new BenchmarkSettings(TestMode.Measurement, RunMode.Iterations, 10, 1000,
new[] { gcBenchmark }, new[] { memoryBenchmark }, new[] { counterBenchmark });
var builder = new BenchmarkBuilder(settings);
var run = builder.NewRun(WarmupData.PreWarmup);
Assert.Equal(2 + (SysInfo.Instance.MaxGcGeneration + 1), run.MeasureCount);
Assert.Equal(1, run.Counters.Count);
Assert.True(run.Counters.ContainsKey(counterBenchmark.CounterName));
}
internal static async Task RunAsync()
{
Console.WriteLine("Initializing container: " + Program.Experiment);
var blobClient = Program.Account.CreateCloudBlobClient();
Container = blobClient.GetContainerReference(Program.Experiment);
if (!(await Container.ExistsAsync()))
{
Console.WriteLine("Container does not exist. Aborting...");
return;
}
Console.WriteLine("Initializing queue: " + Program.Experiment);
var queueClient = Program.Account.CreateCloudQueueClient();
Queue = queueClient.GetQueueReference(Program.Experiment);
if (!(await Queue.ExistsAsync()))
{
Console.WriteLine("Queue does not exist. Aborting...");
return;
}
if (!string.IsNullOrEmpty(Program.OutputDirectory) && !Directory.Exists(Program.OutputDirectory))
{
Directory.CreateDirectory(Program.OutputDirectory);
}
DateTime lastRefresh = DateTime.Now.AddDays(-1);
Console.WriteLine("Worker is running. Press 'a' to abort.");
while (true)
{
bool error = false;
StringBuilder debugInfo = new StringBuilder();
debugInfo.AppendLine(DateTime.Now.ToString());
string debugFileName = null;
if (Console.KeyAvailable && Console.ReadKey().Key == ConsoleKey.A)
{
Console.WriteLine("Aborting...");
return;
}
var msg = await Queue.GetMessageAsync(timeOut, null, null);
if (msg == null)
{
Console.WriteLine("No more tasks in queue.");
return;
}
try
{
Console.WriteLine($"{DateTime.Now}: Loading benchmark...");
Console.WriteLine(msg.AsString);
debugInfo.AppendLine(msg.AsString);
var config = BenchmarkConfig.FromJsonString(msg.AsString);
debugFileName = config.ToShortString();
Console.WriteLine($"{DateTime.Now}: Building benchmark...");
var benchmark = BenchmarkBuilder.Build(config);
var logger = new SimpleConsoleLogger();
logger.Logged += (m, e, l) =>
{
debugInfo.AppendLine(m);
if (e != null)
{
debugInfo.AppendLine(e.ToString());
}
if (l == LogLevel.Error || l == LogLevel.Critical)
{
error = true;
}
};
benchmark.Logger = logger;
Console.WriteLine($"{DateTime.Now}: Starting benchmark...");
var results = benchmark.Execute(true);
Console.WriteLine($"{DateTime.Now}: Generating results...");
string filename = config.ToShortString() + ".xlsx";
if (!string.IsNullOrWhiteSpace(Program.OutputDirectory))
{
filename = Path.Combine(Program.OutputDirectory, filename);
}
benchmark.Dump(filename, config.ToKeyValuePairs());
Console.WriteLine($"{DateTime.Now}: Uploading results...");
string cloudFilename = config.ToShortString() + ".xlsx";
var blob = Container.GetBlockBlobReference(cloudFilename);
await blob.DeleteIfExistsAsync();
await blob.UploadFromFileAsync(filename);
}
catch (Exception exc)
{
debugInfo.AppendLine(exc.ToString());
error = true;
}
if (error)
{
debugFileName = $"{debugFileName ?? Guid.NewGuid().ToString()}.txt";
var cloudFilename = debugFileName;
if (!string.IsNullOrWhiteSpace(Program.OutputDirectory))
{
debugFileName = Path.Combine(Program.OutputDirectory, debugFileName);
}
File.WriteAllText(debugFileName, debugInfo.ToString());
var blob = Container.GetBlockBlobReference(cloudFilename);
await blob.DeleteIfExistsAsync();
await blob.UploadFromFileAsync(debugFileName);
}
await Queue.DeleteMessageAsync(msg);
}
}
