private static IConfig CreateRunConfig(IConfig config, CompetitionAnalyser runState, AccumulationLogger logger)
{
// TODO: better setup?
var result = BenchmarkHelpers.CreateUnitTestConfig(config ?? DefaultConfig.Instance);
result.Add(runState);
result.Add(logger);
result.Add(
StatisticColumn.Min,
ScaledPercentileColumn.S0Column,
ScaledPercentileColumn.S50Column,
ScaledPercentileColumn.S85Column,
ScaledPercentileColumn.S95Column,
ScaledPercentileColumn.S100Column,
StatisticColumn.Max);
return(result);
}
private static Summary RunCore(Type benchmarkType, IConfig runConfig, CompetitionAnalyser runState)
{
Summary summary = null;
const int rerunCount = 10;
for (var i = 0; i < rerunCount; i++)
{
runState.LastRun = i == rerunCount - 1;
runState.RerunRequested = false;
// Running the benchmark
summary = BenchmarkRunner.Run(benchmarkType, runConfig);
// Rerun if annotated
if (!runState.RerunRequested)
{
break;
}
}
return(summary);
}
private static void RunCompetitionUnderSetup(
Type benchmarkType, IConfig config, double minRatio, double maxRatio)
{
ValidateCompetitionSetup(benchmarkType);
// Capturing the output
var logger = InitAccumulationLogger();
// Competition analyzer
var competitionAnalyser = new CompetitionAnalyser();
// Final config
var runConfig = CreateRunConfig(config, competitionAnalyser, logger);
Summary summary = null;
try
{
summary = RunCore(benchmarkType, runConfig, competitionAnalyser);
}
finally
{
DumpOutputSummaryAtTop(summary, logger);
}
competitionAnalyser.ValidateSummary(summary, minRatio, maxRatio);
}
private void AnnotateBenchmarkFiles(
Summary summary, CompetitionAnalyser competitionAnalyser,
ILogger logger, List <IWarning> warnings)
{
var annContext = new AnnotateContext();
var competitionTargets = competitionAnalyser.GetCompetitionTargets(summary);
var newTargets = competitionAnalyser.GetNewCompetitionTargets(summary);
if (newTargets.Length == 0)
{
logger.WriteLineInfo("All competition benchmarks are in boundary.");
return;
}
foreach (var newTarget in newTargets)
{
var targetMethodName = newTarget.CandidateName;
logger.WriteLineInfo($"Method {targetMethodName}: new boundary [{newTarget.MinText},{newTarget.MaxText}].");
int firstCodeLine;
string fileName;
bool hasSource = TryGetSourceInfo(newTarget, out fileName, out firstCodeLine);
if (!hasSource)
{
throw new InvalidOperationException($"Method {targetMethodName}: could not annotate. Source file not found.");
}
if (newTarget.UsesResourceAnnotation)
{
var resourceFileName = Path.ChangeExtension(fileName, ".xml");
logger.WriteLineInfo(
$"Method {targetMethodName}: annotate resource file {resourceFileName}.");
bool annotated = TryFixBenchmarkResource(annContext, resourceFileName, newTarget);
if (!annotated)
{
throw new InvalidOperationException(
$"Method {targetMethodName}: could not annotate resource file {resourceFileName}.");
}
}
else
{
logger.WriteLineInfo($"Method {targetMethodName}: annotate at line {firstCodeLine}, file {fileName}.");
bool annotated = TryFixBenchmarkAttribute(annContext, fileName, firstCodeLine, newTarget);
if (!annotated)
{
throw new InvalidOperationException($"Method {targetMethodName}: could not annotate. Source file {fileName}.");
}
}
if (RerunIfModified && !competitionAnalyser.LastRun)
{
var message = $"Method {targetMethodName} annotation updated, benchmark has to be restarted";
logger.WriteLineInfo(message);
warnings.Add(new Warning(nameof(AnnotateSourceAnalyser), message, null));
competitionTargets[newTarget.Target] = newTarget;
competitionAnalyser.RerunRequested = true;
}
else
{
var message = $"Method {targetMethodName} annotation updated.";
logger.WriteLineInfo(message);
warnings.Add(new Warning(nameof(AnnotateSourceAnalyser), message, null));
}
}
annContext.Save();
}
