private static bool CheckPreconditions(
[NotNull] Type benchmarkType,
bool lockTaken,
CompetitionState competitionState)
{
var runOptions = competitionState.Options.RunOptions;
if (!lockTaken)
{
switch (runOptions.Concurrent)
{
case ConcurrentRunBehavior.Lock:
case ConcurrentRunBehavior.Skip:
competitionState.WriteMessage(
MessageSource.Runner, MessageSeverity.Warning,
"Competition run skipped. Competitions cannot be run in parallel, be sure to disable parallel test execution.");
return(false);
case ConcurrentRunBehavior.Fail:
competitionState.WriteMessage(
MessageSource.Runner, MessageSeverity.SetupError,
"Competition run failed. Competitions cannot be run in parallel, be sure to disable parallel test execution.");
return(false);
default:
throw CodeExceptions.UnexpectedArgumentValue(nameof(runOptions.Concurrent), runOptions.Concurrent);
}
}
if (!runOptions.AllowDebugBuilds && benchmarkType.Assembly.IsDebugAssembly())
{
var assembly = benchmarkType.Assembly;
competitionState.WriteMessage(
MessageSource.Runner, MessageSeverity.Warning,
$"Competition run skipped. Assembly {assembly.GetName().Name} was build as debug.");
return(false);
}
if (runOptions.ContinuousIntegrationMode)
{
competitionState.WriteMessage(
MessageSource.Runner, MessageSeverity.Informational,
"Competition is run under continuous integration service.");
}
return(true);
}
private void ProcessRunComplete(
[NotNull] ICompetitionConfig competitionConfig,
[NotNull] CompetitionState competitionState)
{
var logger = competitionState.Logger;
var summary = competitionState.LastRunSummary;
if (logger == null)
{
return;
}
if (competitionConfig.DetailedLogging)
{
var messages = competitionState.GetMessages();
if (messages.Any())
{
logger.WriteSeparatorLine("All messages");
foreach (var message in messages)
{
logger.LogMessage(message);
}
}
else
{
logger.WriteSeparatorLine();
logger.WriteLineInfo("{LogVerbosePrefix} No messages in run.");
}
}
else if (summary != null)
{
using (Loggers.HostLogger.BeginLogImportant(summary.Config))
{
var summarylogger = DumpSummaryToHostLogger
? logger
: new CompositeLogger(
summary.Config
.GetLoggers()
.Where(l => !(l is HostLogger))
.ToArray());
// Dumping the benchmark results to console
summarylogger.WriteSeparatorLine("Summary");
MarkdownExporter.Console.ExportToLog(summary, summarylogger);
}
}
}
private CompetitionState RunCore(
[NotNull] Type benchmarkType,
[CanBeNull] ICompetitionConfig competitionConfig,
[CanBeNull] CompetitionFeatures competitionFeatures)
{
Code.NotNull(benchmarkType, nameof(benchmarkType));
competitionConfig = CreateBenchmarkConfig(benchmarkType, competitionConfig, competitionFeatures);
var hostLogger = competitionConfig.GetLoggers().OfType <HostLogger>().Single();
var previousDirectory = Environment.CurrentDirectory;
var currentDirectory = GetOutputDirectory(benchmarkType.Assembly);
if (currentDirectory == previousDirectory)
{
currentDirectory = null;
previousDirectory = null;
}
CompetitionState competitionState = null;
try
{
SetCurrentDirectoryIfNotNull(currentDirectory);
try
{
competitionState = CompetitionCore.Run(benchmarkType, competitionConfig);
ProcessRunComplete(competitionState);
}
finally
{
ReportHostLogger(hostLogger, competitionState?.LastRunSummary);
}
using (LoggerHelpers.BeginImportantLogScope(competitionState.Config))
{
ReportMessagesToUser(competitionState);
}
}
finally
{
LoggerHelpers.FlushLoggers(competitionConfig);
SetCurrentDirectoryIfNotNull(previousDirectory);
}
return(competitionState);
}
private static void WriteValidationMessages(CompetitionState competitionState)
{
if (competitionState.LastRunSummary == null)
{
return;
}
foreach (var validationError in competitionState.LastRunSummary.ValidationErrors)
{
var severity = validationError.IsCritical ? MessageSeverity.SetupError : MessageSeverity.Warning;
var message = validationError.Benchmark == null
? validationError.Message
: $"Benchmark {validationError.Benchmark.ShortInfo}:{Environment.NewLine}\t{validationError.Message}";
competitionState.WriteMessage(MessageSource.Validator, severity, message);
}
}
private static CompetitionState InitCompetitionState(
Type benchmarkType,
ICompetitionConfig competitionConfig)
{
var competitionState = new CompetitionState(benchmarkType, competitionConfig);
var runStateSlots = competitionState.Config.GetValidators().OfType <RunStateSlots>().ToArray();
if (runStateSlots.Length != 1)
{
throw CodeExceptions.Argument(
nameof(competitionState),
$"The competition state config should include single instance of {nameof(RunStateSlots)} validator.");
}
runStateSlots[0].InitSlot(RunState, competitionState);
return(competitionState);
}
private static bool CheckPreconditions(
CompetitionState competitionState, bool lockTaken,
IMessageLogger messageLogger)
{
var runOptions = competitionState.Options.RunOptions;
if (!lockTaken)
{
switch (runOptions.Concurrent)
{
case ConcurrentRunBehavior.Lock:
case ConcurrentRunBehavior.Skip:
messageLogger.WriteWarningMessage(
"Competition run skipped. Competitions cannot be run in parallel, be sure to disable parallel test execution.");
return(false);
case ConcurrentRunBehavior.Fail:
messageLogger.WriteSetupErrorMessage(
"Competition run failed. Competitions cannot be run in parallel, be sure to disable parallel test execution.");
return(false);
default:
throw CodeExceptions.UnexpectedArgumentValue(nameof(runOptions.Concurrent), runOptions.Concurrent);
}
}
var benchmarkAssembly = competitionState.BenchmarkType.Assembly;
if (!runOptions.AllowDebugBuilds && benchmarkAssembly.IsDebugAssembly())
{
messageLogger.WriteWarningMessage(
$"Competition run skipped. Assembly {benchmarkAssembly.GetName().Name} was build as debug.");
return(false);
}
if (runOptions.ContinuousIntegrationMode)
{
messageLogger.WriteInfoMessage(
"Competition is run under continuous integration service.");
}
return(true);
}
/// <summary>Reports failed assertions to user.</summary> /// <param name="messages">The messages to report.</param> /// <param name="competitionState">State of the run.</param> protected override void ReportAssertionsFailed(string messages, CompetitionState competitionState) => Assert.Fail(messages);
/// <summary>Reports the execution errors to user.</summary> /// <param name="messages">The messages to report.</param> /// <param name="competitionState">State of the run.</param> protected override void ReportExecutionErrors(string messages, CompetitionState competitionState) => Assert.Fail(messages);
private static void RunCore(Type benchmarkType, CompetitionState competitionState)
{
var logger = competitionState.Logger;
var runOptions = competitionState.Options.RunOptions;
Code.InRange(
runOptions.MaxRunsAllowed,
CompetitionRunMode.MaxRunsAllowedCharacteristic.FullId,
0, MaxRunLimit);
while (competitionState.RunsLeft > 0)
{
competitionState.PrepareForRun();
var run = competitionState.RunNumber;
var runsExpected = competitionState.RunNumber + competitionState.RunsLeft;
var runMessage = competitionState.RunLimitExceeded
? $"Run {run}, total runs (expected): {runsExpected} (rerun limit exceeded, last run)"
: $"Run {run}, total runs (expected): {runsExpected}";
using (BeginLogImportant(competitionState.Config))
{
logger.WriteSeparatorLine(runMessage);
}
// TODO: toolchainProvider to base (???).
Func <Job, IToolchain> toolchainProvider = j => j.Infrastructure?.Toolchain ?? InProcessToolchain.Instance;
// Running the benchmark
var summary = BenchmarkRunnerCore.Run(
BenchmarkConverter.TypeToBenchmarks(benchmarkType, competitionState.Config),
competitionState.Config,
toolchainProvider);
competitionState.RunCompleted(summary);
// Dump messages if analysis was not run and there is a validation analyser.
if (summary.HasCriticalValidationErrors)
{
var validationAnalyser = competitionState.Config
.GetAnalysers()
.OfType <ValidatorMessagesAnalyser>()
.FirstOrDefault();
validationAnalyser?.Analyse(summary);
}
if (competitionState.HasCriticalErrorsInRun)
{
competitionState.Logger.WriteVerboseHint("Breaking competition execution. High severity error occured.");
break;
}
if (competitionState.RunLimitExceeded)
{
break;
}
if (competitionState.RunsLeft > 0)
{
competitionState.Logger.WriteVerboseHint($"Rerun requested. Runs left: {competitionState.RunsLeft}.");
}
}
if (competitionState.RunLimitExceeded && competitionState.RunsLeft > 0)
{
competitionState.WriteMessage(
MessageSource.Runner, MessageSeverity.TestError,
$"The benchmark run limit ({runOptions.MaxRunsAllowed} runs(s)) exceeded, check log for details.");
}
else if (competitionState.RunNumber > 1)
{
competitionState.WriteMessage(
MessageSource.Runner, MessageSeverity.Warning,
$"The benchmark was run {competitionState.RunNumber} time(s), check log for details.");
}
}
private void ReportMessagesToUser(
[NotNull] CompetitionState competitionState)
{
var criticalErrorMessages = GetMessageLines(
competitionState, m => m.MessageSeverity > MessageSeverity.TestError, true);
var hasCriticalMessages = criticalErrorMessages.Any();
var testFailedMessages = GetMessageLines(
competitionState, m => m.MessageSeverity == MessageSeverity.TestError, hasCriticalMessages);
var hasTestFailedMessages = testFailedMessages.Any();
var warningMessages = GetMessageLines(
competitionState, m => m.MessageSeverity == MessageSeverity.Warning, true);
var hasWarnings = warningMessages.Any();
var infoMessages = GetMessageLines(competitionState, m => m.MessageSeverity < MessageSeverity.Warning, true);
var hasInfo = infoMessages.Any();
if (!(hasCriticalMessages || hasTestFailedMessages || hasWarnings))
{
return;
}
var allMessages = new List <string>();
// TODO: simplify
if (hasCriticalMessages)
{
allMessages.Add("Test completed with errors, details below.");
}
else if (hasTestFailedMessages)
{
allMessages.Add("Test failed, details below.");
}
else
{
allMessages.Add("Test completed with warnings, details below.");
}
if (hasCriticalMessages)
{
allMessages.Add("Errors:");
allMessages.AddRange(criticalErrorMessages);
}
if (hasTestFailedMessages)
{
allMessages.Add("Failed assertions:");
allMessages.AddRange(testFailedMessages);
}
if (hasWarnings)
{
allMessages.Add("Warnings:");
allMessages.AddRange(warningMessages);
}
if (hasInfo)
{
allMessages.Add("Diagnostic messages:");
allMessages.AddRange(infoMessages);
}
var messageText = allMessages.Join(Environment.NewLine);
var runOptions = competitionState.Options.RunOptions;
if (hasCriticalMessages)
{
ReportExecutionErrors(messageText, competitionState);
}
else if (hasTestFailedMessages || runOptions.ReportWarningsAsErrors)
{
ReportAssertionsFailed(messageText, competitionState);
}
else
{
ReportWarnings(messageText, competitionState);
}
}
/// <summary>Reports warnings to user.</summary>
/// <param name="messages">The messages to report.</param>
/// <param name="competitionState">State of the run.</param>
protected override void ReportWarnings(string messages, CompetitionState competitionState)
{
throw new SkipTestException(messages);
}
/// <summary>Reports the execution errors to user.</summary>
/// <param name="messages">The messages to report.</param>
/// <param name="competitionState">State of the run.</param>
protected override void ReportExecutionErrors(string messages, CompetitionState competitionState)
{
throw new XunitException(messages + Environment.NewLine);
}
/// <summary>Reports the execution errors to user.</summary> /// <param name="messages">The messages to report.</param> /// <param name="competitionState">State of the run.</param> protected abstract void ReportExecutionErrors([NotNull] string messages, [NotNull] CompetitionState competitionState);
private static void RunCore(CompetitionState competitionState, IMessageLogger messageLogger)
{
var logger = competitionState.Logger;
var runOptions = competitionState.Options.RunOptions;
Code.InRange(
runOptions.MaxRunsAllowed,
CompetitionRunMode.MaxRunsAllowedCharacteristic.FullId,
0, MaxRunLimit);
while (competitionState.RunsLeft > 0)
{
competitionState.PrepareForRun();
LogCompetitionRunHeader(competitionState);
// Running the benchmark
var benchmarkType = competitionState.BenchmarkType;
var runInfo = _typeToBenchmarkHack(
benchmarkType, benchmarkType.GetMethods(), new ReadOnlyConfig(competitionState.Config));
var summary = BenchmarkRunnerCore.Run(
runInfo,
j => j.Infrastructure?.Toolchain ?? InProcessToolchain.Instance);
competitionState.RunCompleted(summary);
// Dump messages if analysis was not run and there is a validation analyser.
if (summary.HasCriticalValidationErrors)
{
var validationAnalyser = competitionState.Config
.GetAnalysers()
.OfType <ValidatorMessagesAnalyser>()
.FirstOrDefault();
validationAnalyser?.Analyse(summary);
}
if (competitionState.HasCriticalErrorsInRun)
{
logger.WriteHintLine("Breaking competition execution. High severity error occured.");
break;
}
if (competitionState.RunLimitExceeded)
{
break;
}
if (competitionState.RunsLeft > 0)
{
logger.WriteHintLine($"Rerun requested. Runs left: {competitionState.RunsLeft}.");
}
}
if (competitionState.RunLimitExceeded && competitionState.RunsLeft > 0)
{
messageLogger.WriteTestErrorMessage(
$"The benchmark run limit ({runOptions.MaxRunsAllowed} runs(s)) exceeded, check log for details.");
}
else if (competitionState.RunNumber > 1)
{
messageLogger.WriteWarningMessage(
$"The benchmark was run {competitionState.RunNumber} time(s), check log for details.");
}
}
/// <summary>Reports warnings to user.</summary> /// <param name="messages">The messages to report.</param> /// <param name="competitionState">State of the run.</param> protected override void ReportWarnings(string messages, CompetitionState competitionState) => Assert.Ignore(messages);
/// <summary>Reports warnings to user.</summary>
/// <param name="messages">The messages to report.</param>
/// <param name="competitionState">State of the run.</param>
protected override void ReportWarnings(string messages, CompetitionState competitionState)
{
throw new SkipTestException(messages + Environment.NewLine);
}
/// <summary>Reports failed assertions to user.</summary> /// <param name="messages">The messages to report.</param> /// <param name="competitionState">State of the run.</param> protected abstract void ReportAssertionsFailed([NotNull] string messages, [NotNull] CompetitionState competitionState);
/// <summary>Reports failed assertions to user.</summary>
/// <param name="messages">The messages to report.</param>
/// <param name="competitionState">State of the run.</param>
protected override void ReportAssertionsFailed(string messages, CompetitionState competitionState)
{
throw new XunitException(messages + Environment.NewLine);
}
/// <summary>Reports warnings to user.</summary> /// <param name="messages">The messages to report.</param> /// <param name="competitionState">State of the run.</param> protected abstract void ReportWarnings([NotNull] string messages, [NotNull] CompetitionState competitionState);