private static IConfig CreateRunConfig(IConfig config, CompetitionAnalyser runState, AccumulationLogger logger) { // TODO: better setup? var result = BenchmarkHelpers.CreateUnitTestConfig(config ?? DefaultConfig.Instance); result.Add(runState); result.Add(logger); result.Add( StatisticColumn.Min, ScaledPercentileColumn.S0Column, ScaledPercentileColumn.S50Column, ScaledPercentileColumn.S85Column, ScaledPercentileColumn.S95Column, ScaledPercentileColumn.S100Column, StatisticColumn.Max); return(result); }
private static Summary RunCore(Type benchmarkType, IConfig runConfig, CompetitionAnalyser runState) { Summary summary = null; const int rerunCount = 10; for (var i = 0; i < rerunCount; i++) { runState.LastRun = i == rerunCount - 1; runState.RerunRequested = false; // Running the benchmark summary = BenchmarkRunner.Run(benchmarkType, runConfig); // Rerun if annotated if (!runState.RerunRequested) { break; } } return(summary); }
private static void RunCompetitionUnderSetup( Type benchmarkType, IConfig config, double minRatio, double maxRatio) { ValidateCompetitionSetup(benchmarkType); // Capturing the output var logger = InitAccumulationLogger(); // Competition analyzer var competitionAnalyser = new CompetitionAnalyser(); // Final config var runConfig = CreateRunConfig(config, competitionAnalyser, logger); Summary summary = null; try { summary = RunCore(benchmarkType, runConfig, competitionAnalyser); } finally { DumpOutputSummaryAtTop(summary, logger); } competitionAnalyser.ValidateSummary(summary, minRatio, maxRatio); }
private void AnnotateBenchmarkFiles( Summary summary, CompetitionAnalyser competitionAnalyser, ILogger logger, List <IWarning> warnings) { var annContext = new AnnotateContext(); var competitionTargets = competitionAnalyser.GetCompetitionTargets(summary); var newTargets = competitionAnalyser.GetNewCompetitionTargets(summary); if (newTargets.Length == 0) { logger.WriteLineInfo("All competition benchmarks are in boundary."); return; } foreach (var newTarget in newTargets) { var targetMethodName = newTarget.CandidateName; logger.WriteLineInfo($"Method {targetMethodName}: new boundary [{newTarget.MinText},{newTarget.MaxText}]."); int firstCodeLine; string fileName; bool hasSource = TryGetSourceInfo(newTarget, out fileName, out firstCodeLine); if (!hasSource) { throw new InvalidOperationException($"Method {targetMethodName}: could not annotate. Source file not found."); } if (newTarget.UsesResourceAnnotation) { var resourceFileName = Path.ChangeExtension(fileName, ".xml"); logger.WriteLineInfo( $"Method {targetMethodName}: annotate resource file {resourceFileName}."); bool annotated = TryFixBenchmarkResource(annContext, resourceFileName, newTarget); if (!annotated) { throw new InvalidOperationException( $"Method {targetMethodName}: could not annotate resource file {resourceFileName}."); } } else { logger.WriteLineInfo($"Method {targetMethodName}: annotate at line {firstCodeLine}, file {fileName}."); bool annotated = TryFixBenchmarkAttribute(annContext, fileName, firstCodeLine, newTarget); if (!annotated) { throw new InvalidOperationException($"Method {targetMethodName}: could not annotate. Source file {fileName}."); } } if (RerunIfModified && !competitionAnalyser.LastRun) { var message = $"Method {targetMethodName} annotation updated, benchmark has to be restarted"; logger.WriteLineInfo(message); warnings.Add(new Warning(nameof(AnnotateSourceAnalyser), message, null)); competitionTargets[newTarget.Target] = newTarget; competitionAnalyser.RerunRequested = true; } else { var message = $"Method {targetMethodName} annotation updated."; logger.WriteLineInfo(message); warnings.Add(new Warning(nameof(AnnotateSourceAnalyser), message, null)); } } annContext.Save(); }