private static CompetitionState InitCompetitionState(
Type benchmarkType,
ICompetitionConfig competitionConfig)
{
var competitionState = new CompetitionState(benchmarkType, competitionConfig);
var runStateSlots = competitionState.Config.GetValidators().OfType <RunStateSlots>().ToArray();
if (runStateSlots.Length != 1)
{
throw CodeExceptions.Argument(
nameof(competitionState),
$"The competition state config should include single instance of {nameof(RunStateSlots)} validator.");
}
runStateSlots[0].InitSlot(RunState, competitionState);
return(competitionState);
}
/// <summary>Initializes a new instance of the <see cref="CompetitionState"/> class.</summary>
internal CompetitionState(
[NotNull] Type benchmarkType,
[NotNull] ICompetitionConfig competitionConfig)
{
Code.NotNull(benchmarkType, nameof(benchmarkType));
Code.NotNull(competitionConfig, nameof(competitionConfig));
SummaryFromAllRuns = _summaries.AsReadOnly();
BenchmarkType = benchmarkType;
Config = competitionConfig;
Logger = competitionConfig.GetCompositeLogger();
RunNumber = 0;
RunsLeft = 1;
HighestMessageSeverityInRun = MessageSeverity.Verbose;
HighestMessageSeverity = MessageSeverity.Verbose;
LastRunSummary = null;
}
private IConfig CreateBenchmarkConfig(ICompetitionConfig competitionConfig)
{
// ReSharper disable once UseObjectOrCollectionInitializer
var result = new ManualConfig();
// TODO: to overridable methods?
result.KeepBenchmarkFiles = competitionConfig.KeepBenchmarkFiles;
result.UnionRule = competitionConfig.UnionRule;
result.Set(competitionConfig.GetOrderProvider());
result.Add(OverrideJobs(competitionConfig).ToArray());
result.Add(OverrideExporters(competitionConfig).ToArray());
result.Add(OverrideDiagnosers(competitionConfig).ToArray());
result.Add(GetLoggers(competitionConfig).ToArray());
result.Add(GetColumns(competitionConfig).ToArray());
result.Add(GetValidators(competitionConfig).ToArray());
result.Add(GetAnalysers(competitionConfig).ToArray());
return(result.AsReadOnly());
}
private void AddCompetitionProperties([NotNull] ICompetitionConfig config)
{
//Runner config - troubleshooting
AllowDebugBuilds = config.AllowDebugBuilds;
DetailedLogging = config.DetailedLogging;
// Runner config
MaxRunsAllowed = config.MaxRunsAllowed;
ReportWarningsAsErrors = config.ReportWarningsAsErrors;
ConcurrentRunBehavior = config.ConcurrentRunBehavior;
// Validation config
IgnoreExistingAnnotations = config.IgnoreExistingAnnotations;
AllowLongRunningBenchmarks = config.AllowLongRunningBenchmarks;
RerunIfLimitsFailed = config.RerunIfLimitsFailed;
LogCompetitionLimits = config.LogCompetitionLimits;
CompetitionLimitProvider = config.CompetitionLimitProvider;
// Annotation config
UpdateSourceAnnotations = config.UpdateSourceAnnotations;
PreviousRunLogUri = config.PreviousRunLogUri;
}
private ICompetitionConfig PrepareCompetitionConfig(
[NotNull] Type benchmarkType,
[CanBeNull] ICompetitionConfig competitionConfig)
{
var result = GetFirstConfig(benchmarkType, competitionConfig);
if (result.CompetitionLimitProvider == null)
{
result.CompetitionLimitProvider = LogNormalLimitProvider.Instance;
}
if (result.MaxRunsAllowed <= 0)
{
result.MaxRunsAllowed = DefaultMaxRunsAllowed;
}
if (EnvironmentInfo.GetCurrent().HasAttachedDebugger)
{
result.AllowDebugBuilds = true;
}
return(result.AsReadOnly());
}
public CompetitionState Run(
[NotNull] Type benchmarkType,
[NotNull] ICompetitionConfig competitionConfig) =>
RunCore(benchmarkType, competitionConfig, null);
public CompetitionState Run <T>(
[NotNull] T thisReference,
[NotNull] ICompetitionConfig competitionConfig)
where T : class =>
RunCore(thisReference.GetType(), competitionConfig, null);
/// <summary>Override competition exporters.</summary> /// <param name="competitionConfig">The competition config.</param> /// <returns>The jobs for the competition</returns> protected virtual List <IExporter> OverrideExporters([NotNull] ICompetitionConfig competitionConfig) => competitionConfig.GetExporters().ToList();
// TODO: HACK: Remove after update to BDN 10.4 /// <summary>Runs the competition - core implementation.</summary> /// <param name="benchmarkType">Benchmark class to run.</param> /// <param name="competitionConfig">The competition config.</param> /// <returns>Competition state for the run.</returns> protected virtual CompetitionState RunCore(Type benchmarkType, ICompetitionConfig competitionConfig) => CompetitionCore.Run(benchmarkType, competitionConfig);
/// <summary>Override competition loggers.</summary> /// <param name="competitionConfig">The competition config.</param> /// <returns>The loggers for the competition</returns> protected virtual List <ILogger> OverrideLoggers([NotNull] ICompetitionConfig competitionConfig) => competitionConfig.GetLoggers().ToList();
/// <summary>Applies job modifier.</summary>
/// <param name="config">The config.</param>
/// <param name="jobModifier">Job modifier to apply.</param>
/// <returns>Config with applied job modifier.</returns>
public static ICompetitionConfig WithModifier(
this ICompetitionConfig config, Job jobModifier) =>
config.With(m => m.ApplyModifier(jobModifier));
internal static CompetitionState Run(
[NotNull] Type benchmarkType,
[NotNull] ICompetitionConfig competitionConfig)
{
Code.NotNull(benchmarkType, nameof(benchmarkType));
Code.NotNull(competitionConfig, nameof(competitionConfig));
var runStateSlots = competitionConfig.GetValidators().OfType <RunStateSlots>();
if (runStateSlots.Count() != 1)
{
throw CodeExceptions.Argument(
nameof(competitionConfig),
$"The competition config should include single instance of {nameof(RunStateSlots)} validator.");
}
var competitionState = RunState[competitionConfig];
try
{
competitionState.FirstTimeInit(benchmarkType, competitionConfig);
var logger = competitionState.Logger;
using (BeginLogImportant(competitionConfig))
{
logger.WriteSeparatorLine(benchmarkType.Name, true);
logger.WriteLineHelp($"{LogInfoPrefix} {benchmarkType.GetShortAssemblyQualifiedName()}");
}
using (var mutex = new Mutex(false, $"Global\\{typeof(CompetitionCore).FullName}"))
{
var lockTaken = false;
try
{
var timeout = competitionState.Options.RunOptions.Concurrent == ConcurrentRunBehavior.Lock
? TotalWaitTimeout
: TimeSpan.Zero;
lockTaken = SpinWait(mutex, timeout, SpinWaitRunTimeout, competitionState);
if (CheckPreconditions(benchmarkType, lockTaken, competitionState))
{
RunCore(benchmarkType, competitionState);
}
}
finally
{
if (lockTaken)
{
mutex.ReleaseMutex();
}
}
}
}
catch (TargetInvocationException ex)
{
competitionState.WriteExceptionMessage(
MessageSource.Runner, MessageSeverity.ExecutionError,
$"Benchmark {benchmarkType.Name} failed.", ex.InnerException ?? ex);
}
catch (Exception ex)
{
competitionState.WriteExceptionMessage(
MessageSource.Runner, MessageSeverity.ExecutionError,
$"Benchmark {benchmarkType.Name} failed.", ex);
}
finally
{
LoggerHelpers.FlushLoggers(competitionConfig);
}
competitionState.CompetitionCompleted();
return(competitionState);
}
public static int GetExpectedCountIgnoreUnroll(ICompetitionConfig config, int methodsCount)
{
return(GetExpectedCountCore(config, methodsCount, 1));
}
/// <summary>Initializes a new instance of the <see cref="ReadOnlyCompetitionConfig"/> class.</summary>
/// <param name="config">The config to wrap.</param>
public ReadOnlyCompetitionConfig([NotNull] ICompetitionConfig config) : base(config)
{
Code.NotNull(config, nameof(config));
_config = config;
}
public CompetitionState Run <T>([CanBeNull] ICompetitionConfig competitionConfig = null)
where T : class =>
RunCore(typeof(T), competitionConfig);
private void ReportMessagesToUser(
[NotNull] ICompetitionConfig competitionConfig,
[NotNull] CompetitionState competitionState)
{
var criticalErrorMessages = GetMessageLines(
competitionState, m => m.MessageSeverity > MessageSeverity.TestError, true);
bool hasCriticalMessages = criticalErrorMessages.Any();
var testFailedMessages = GetMessageLines(
competitionState, m => m.MessageSeverity == MessageSeverity.TestError, hasCriticalMessages);
bool hasTestFailedMessages = testFailedMessages.Any();
var warningMessages = GetMessageLines(
competitionState, m => m.MessageSeverity == MessageSeverity.Warning, true);
bool hasWarnings = warningMessages.Any();
var infoMessages = GetMessageLines(competitionState, m => m.MessageSeverity < MessageSeverity.Warning, true);
bool hasInfo = infoMessages.Any();
if (!(hasCriticalMessages || hasTestFailedMessages || hasWarnings))
{
return;
}
var allMessages = new List <string>();
// TODO: simplify
if (hasCriticalMessages)
{
allMessages.Add("Test completed with errors, details below.");
}
else if (hasTestFailedMessages)
{
allMessages.Add("Test failed, details below.");
}
else
{
allMessages.Add("Test completed with warnings, details below.");
}
if (hasCriticalMessages)
{
allMessages.Add("Errors:");
allMessages.AddRange(criticalErrorMessages);
}
if (hasTestFailedMessages)
{
allMessages.Add("Failed assertions:");
allMessages.AddRange(testFailedMessages);
}
if (hasWarnings)
{
allMessages.Add("Warnings:");
allMessages.AddRange(warningMessages);
}
if (hasInfo)
{
allMessages.Add("Diagnostic messages:");
allMessages.AddRange(infoMessages);
}
var messageText = string.Join(Environment.NewLine, allMessages);
if (hasCriticalMessages)
{
ReportExecutionErrors(messageText, competitionState);
}
else if (hasTestFailedMessages || competitionConfig.ReportWarningsAsErrors)
{
ReportAssertionsFailed(messageText, competitionState);
}
else
{
ReportWarnings(messageText, competitionState);
}
}
/// <summary>Override competition analysers.</summary> /// <param name="competitionConfig">The competition config.</param> /// <returns>The analysers for the competition</returns> protected virtual List <IAnalyser> OverrideAnalysers([NotNull] ICompetitionConfig competitionConfig) => competitionConfig.GetAnalysers().ToList();
/// <summary>Override competition validators.</summary> /// <param name="competitionConfig">The competition config.</param> /// <returns>The validators for the competition</returns> protected virtual List <IValidator> OverrideValidators([NotNull] ICompetitionConfig competitionConfig) => competitionConfig.GetValidators().ToList();
/// <summary>Override competition columns.</summary> /// <param name="competitionConfig">The competition config.</param> /// <returns>The columns for the competition</returns> protected virtual List <IColumn> OverrideColumns([NotNull] ICompetitionConfig competitionConfig) => competitionConfig.GetColumns().ToList();
public IReadOnlyDictionary <Type, CompetitionState> Run(
[NotNull] Assembly assembly,
[NotNull] ICompetitionConfig competitionConfig) =>
Run(BenchmarkHelpers.GetBenchmarkTypes(assembly), competitionConfig);
public CompetitionState Run <T>([NotNull] ICompetitionConfig competitionConfig)
where T : class =>
RunCore(typeof(T), competitionConfig, null);
public static CompetitionState Run <T>(
[NotNull] T thisReference,
[CanBeNull] ICompetitionConfig competitionConfig = null)
where T : class =>
Runner.Run(thisReference, competitionConfig);
/// <summary>Applies the specified competition options.</summary>
/// <param name="config">The config.</param>
/// <param name="optionsModifier">Competition options to apply.</param>
/// <returns>Config with applied competition options.</returns>
public static ICompetitionConfig WithModifier(
this ICompetitionConfig config, CompetitionOptions optionsModifier) =>
config.With(m => m.ApplyModifier(optionsModifier));
public static IReadOnlyDictionary <Type, CompetitionState> Run(
[NotNull] Assembly assembly,
[CanBeNull] ICompetitionConfig competitionConfig = null) =>
Runner.Run(assembly, competitionConfig);
public static CompetitionState Run <T>([CanBeNull] ICompetitionConfig competitionConfig = null)
where T : class =>
Runner.Run <T>(competitionConfig);
/// <summary>Override competition diagnosers.</summary> /// <param name="competitionConfig">The competition config.</param> /// <returns>The diagnosers for the competition</returns> protected virtual List <IDiagnoser> OverrideDiagnosers([NotNull] ICompetitionConfig competitionConfig) => competitionConfig.GetDiagnosers().ToList();
public static CompetitionState Run(
[NotNull] Type benchmarkType,
[CanBeNull] ICompetitionConfig competitionConfig = null) =>
Runner.Run(benchmarkType, competitionConfig);
/// <summary>Initializes a new instance of the <see cref="ManualCompetitionConfig"/> class.</summary>
/// <param name="config">The config to init from.</param>
public ManualCompetitionConfig([CanBeNull] ICompetitionConfig config)
{
Add(config);
}
public static IReadOnlyDictionary <Type, CompetitionState> Run(
[NotNull] Type[] benchmarkTypes,
[CanBeNull] ICompetitionConfig competitionConfig = null) =>
Runner.Run(benchmarkTypes, competitionConfig);
public static CompetitionState Run <T>([NotNull] ICompetitionConfig competitionConfig)
where T : class =>
Runner.Run <T>(competitionConfig);
