public static void CompetitionNoBaselineFailBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <NoBaselineFailBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
AssertCompetitionCompleted(runState, MessageSeverity.SetupError);
Assert.AreEqual(messages.Length, 2);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.Informational);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Analyser);
Assert.AreEqual(
messages[0].MessageText,
"Target Benchmark1. Metric validation skipped as the method is not marked with CompetitionBenchmarkAttribute.");
Assert.AreEqual(messages[1].RunNumber, 1);
Assert.AreEqual(messages[1].RunMessageNumber, 2);
Assert.AreEqual(messages[1].MessageSeverity, MessageSeverity.SetupError);
Assert.AreEqual(messages[1].MessageSource, MessageSource.Analyser);
Assert.AreEqual(
messages[1].MessageText,
"No baseline member found. Apply CompetitionBaselineAttribute to the one of benchmark methods.");
}
public static void CompetitionTypeLevelConfig()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <BenchmarkWithConfig>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].MessageText, "All competition metrics are ok.");
AssertColumn(runState, BaselineScaledColumn.WelchTTestPValue, 1);
AssertColumn(runState, StatisticColumn.Kurtosis, 0);
AssertColumn(runState, StatisticColumn.Skewness, 1);
Interlocked.Exchange(ref _callCounter, 0);
runState = SelfTestCompetition.Run <Nested.BenchmarkWithConfig2>();
messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].MessageText, "All competition metrics are ok.");
AssertColumn(runState, BaselineScaledColumn.WelchTTestPValue, 1);
AssertColumn(runState, StatisticColumn.Kurtosis, 1);
AssertColumn(runState, StatisticColumn.Skewness, 1);
Interlocked.Exchange(ref _callCounter, 0);
}
public static void TestIoReadMetricAccuracy()
{
var runState = SelfTestCompetition.Run <IoReadBenchmark>();
var summary = runState.LastRunSummary;
var readBytes = MetricInfo.FromAttribute <FileIoReadAttribute>().ValuesProvider.TryGetMeanValue(summary.Benchmarks[0], summary);
Assert.Greater(readBytes, 0);
}
public static void TestClrExceptionsAccuracy()
{
var runState = SelfTestCompetition.Run <ThrowExceptionsBenchmark>();
var summary = runState.LastRunSummary;
var exceptions = MetricInfo.FromAttribute <ClrExceptionsAttribute>().ValuesProvider.TryGetMeanValue(summary.Benchmarks[0], summary);
//Assert.True(runState.CompletedSuccessfully);
Assert.Greater(exceptions ?? 1, 0);
}
public static void CompetitionEmptyBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <EmptyBenchmark>(SelfTestConfig);
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, 0);
AssertCompetitionCompleted(runState, MessageSeverity.Verbose);
Assert.AreEqual(messages.Length, 0);
}
public static void CompetitionLimitsFailBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <LimitsFailBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, 3 * ExpectedRunCount); // 3x rerun
AssertCompetitionCompleted(runState, MessageSeverity.TestError, runNumber: 3);
Assert.AreEqual(messages.Length, 6);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.TestError);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Analyser);
Assert.That(messages[0].MessageText, Does.StartWith("Target SlowerX10. Metric Scaled"));
Assert.That(messages[0].MessageText, Does.Contain(" is out of limit "));
Assert.AreEqual(messages[1].RunNumber, 1);
Assert.AreEqual(messages[1].RunMessageNumber, 2);
Assert.AreEqual(messages[1].MessageSeverity, MessageSeverity.Informational);
Assert.AreEqual(messages[1].MessageSource, MessageSource.Runner);
Assert.AreEqual(messages[1].MessageText, "Metrics check failed. Requesting 1 run(s).");
Assert.AreEqual(messages[2].RunNumber, 2);
Assert.AreEqual(messages[2].RunMessageNumber, 1);
Assert.AreEqual(messages[2].MessageSeverity, MessageSeverity.TestError);
Assert.AreEqual(messages[2].MessageSource, MessageSource.Analyser);
Assert.That(messages[2].MessageText, Does.StartWith("Target SlowerX10. Metric Scaled"));
Assert.That(messages[2].MessageText, Does.Contain(" is out of limit "));
Assert.AreEqual(messages[3].RunNumber, 2);
Assert.AreEqual(messages[3].RunMessageNumber, 2);
Assert.AreEqual(messages[3].MessageSeverity, MessageSeverity.Informational);
Assert.AreEqual(messages[3].MessageSource, MessageSource.Runner);
Assert.AreEqual(messages[3].MessageText, "Metrics check failed. Requesting 1 run(s).");
Assert.AreEqual(messages[4].RunNumber, 3);
Assert.AreEqual(messages[4].RunMessageNumber, 1);
Assert.AreEqual(messages[4].MessageSeverity, MessageSeverity.TestError);
Assert.AreEqual(messages[4].MessageSource, MessageSource.Analyser);
Assert.That(messages[4].MessageText, Does.StartWith("Target SlowerX10. Metric Scaled"));
Assert.That(messages[4].MessageText, Does.Contain(" is out of limit "));
Assert.AreEqual(messages[5].RunNumber, 3);
Assert.AreEqual(messages[5].RunMessageNumber, 2);
Assert.AreEqual(messages[5].MessageSeverity, MessageSeverity.Warning);
Assert.AreEqual(messages[5].MessageSource, MessageSource.Runner);
Assert.AreEqual(
messages[5].MessageText,
"The benchmark was run 3 time(s), check log for details.");
}
public static void CompetitionLimitsEmptyFailBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <LimitsEmptyFailBenchmark>(SelfTestConfig);
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, 3 * ExpectedRunCount); // 3x rerun
AssertCompetitionCompleted(runState, MessageSeverity.TestError, runNumber: 3);
Assert.AreEqual(messages.Length, 6);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.TestError);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Analyser);
Assert.That(messages[0].MessageText, Does.StartWith("Method SlowerX10"));
Assert.That(messages[0].MessageText, Does.EndWith(" has empty limit. Please fill it."));
Assert.AreEqual(messages[1].RunNumber, 1);
Assert.AreEqual(messages[1].RunMessageNumber, 2);
Assert.AreEqual(messages[1].MessageSeverity, MessageSeverity.Informational);
Assert.AreEqual(messages[1].MessageSource, MessageSource.Runner);
Assert.AreEqual(messages[1].MessageText, "Requesting 1 run(s): Limit checking failed.");
Assert.AreEqual(messages[2].RunNumber, 2);
Assert.AreEqual(messages[2].RunMessageNumber, 1);
Assert.AreEqual(messages[2].MessageSeverity, MessageSeverity.TestError);
Assert.AreEqual(messages[2].MessageSource, MessageSource.Analyser);
Assert.That(messages[2].MessageText, Does.StartWith("Method SlowerX10"));
Assert.That(messages[2].MessageText, Does.EndWith(" has empty limit. Please fill it."));
Assert.AreEqual(messages[3].RunNumber, 2);
Assert.AreEqual(messages[3].RunMessageNumber, 2);
Assert.AreEqual(messages[3].MessageSeverity, MessageSeverity.Informational);
Assert.AreEqual(messages[3].MessageSource, MessageSource.Runner);
Assert.AreEqual(messages[3].MessageText, "Requesting 1 run(s): Limit checking failed.");
Assert.AreEqual(messages[4].RunNumber, 3);
Assert.AreEqual(messages[4].RunMessageNumber, 1);
Assert.AreEqual(messages[4].MessageSeverity, MessageSeverity.TestError);
Assert.AreEqual(messages[4].MessageSource, MessageSource.Analyser);
Assert.That(messages[4].MessageText, Does.StartWith("Method SlowerX10"));
Assert.That(messages[4].MessageText, Does.EndWith(" has empty limit. Please fill it."));
Assert.AreEqual(messages[5].RunNumber, 3);
Assert.AreEqual(messages[5].RunMessageNumber, 2);
Assert.AreEqual(messages[5].MessageSeverity, MessageSeverity.Warning);
Assert.AreEqual(messages[5].MessageSource, MessageSource.Runner);
Assert.AreEqual(
messages[5].MessageText,
"The benchmark was run 3 time(s) (read log for details). Try to loose competition limits.");
}
public static void CompetitionHighAccuracyBenchmark()
{
IgnoreIfDebug();
var runState = SelfTestCompetition.Run <HighAccuracyBenchmark>();
var messages = runState.GetMessages();
if (messages.All(m => m.MessageText != "All competition metrics are ok."))
{
Assert.Ignore("The environment does not provide accurate timings. Test results cannot be trusted.");
}
}
public static void TestAnnotateFromRemoteLog()
{
var runState = SelfTestCompetition.Run <AnnotatedBenchmark>(_remoteLogConfig);
var messages = runState.GetMessages();
Assert.AreEqual(runState.HighestMessageSeverity, MessageSeverity.Warning);
Assert.IsTrue(runState.Completed);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 4);
}
public static void TestAnnotateBaselineChangedFromLocalLog()
{
var runState = SelfTestCompetition.Run <AnnotatedBaselineChangedBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(runState.HighestMessageSeverity, MessageSeverity.SetupError);
Assert.IsTrue(runState.Completed);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 3);
}
public static void TestAnnotateFromLocalLogTimeAndGcNoRelative()
{
var runState = SelfTestCompetition.Run <AnnotatedWithTimeAndGcNoRelativeBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(runState.HighestMessageSeverity, MessageSeverity.Warning);
Assert.IsTrue(runState.Completed);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 4);
}
public static void CompetitionEmptyBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <EmptyBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, 0);
AssertCompetitionCompleted(runState, MessageSeverity.SetupError);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].MessageText, "Nothing to check as there is no methods in benchmark.");
}
public static void CompetitionAssemblyLevelConfig()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <BenchmarkWithoutConfig>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].MessageText, "CompetitionAnalyser: All competition limits are ok.");
Assert.AreEqual(runState.Config.GetColumns().Count(c => c == BaselineDiffColumn.Delta), 1);
Assert.AreEqual(runState.Config.GetColumns().Count(c => c == BaselineDiffColumn.Scaled95), 0);
}
public static void CompetitionXmlFullAnnotationBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <XmlFullAnnotationBenchmark>(SelfTestConfig);
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
AssertCompetitionCompleted(runState, MessageSeverity.Informational);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].MessageText, "CompetitionAnalyser: All competition limits are ok.");
}
public static void CompetitionNoBaselineOkBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <NoBaselineOkBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
AssertCompetitionCompleted(runState, MessageSeverity.Informational);
Assert.AreEqual(messages.Length, 2);
Assert.AreEqual(messages[0].MessageText, "Target Benchmark1. Metric validation skipped as the method is not marked with CompetitionBenchmarkAttribute.");
Assert.AreEqual(messages[1].MessageText, "Target Benchmark2. Metric validation skipped as the method is not marked with CompetitionBenchmarkAttribute.");
}
public static void CompetitionOkBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <OkBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
AssertCompetitionCompleted(runState, MessageSeverity.Informational);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].MessageText, "All competition metrics are ok.");
}
public static void TestAnnotateFromLocalLog()
{
// TODO: message if no XML annotation
// TODO: exact message validation
var runState = SelfTestCompetition.Run <AnnotatedBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(runState.HighestMessageSeverity, MessageSeverity.Warning);
Assert.IsTrue(runState.Completed);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 4);
}
public static void CompetitionXmlTaskOkBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <XmlTaskOkBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, 2 * ExpectedRunCount);
AssertCompetitionCompleted(runState, MessageSeverity.Informational);
Assert.AreEqual(messages.Length, 3);
Assert.AreEqual(messages[0].MessageText, "Target SlowerX2Async. Metric validation skipped as the method is marked with CompetitionBenchmarkAttribute.DoesNotCompete set to true.");
Assert.AreEqual(messages[1].MessageText, "Target SlowerX3Async. Metric validation skipped as the method is not marked with CompetitionBenchmarkAttribute.");
Assert.AreEqual(messages[2].MessageText, "All competition metrics are ok.");
}
public static void CompetitionHighAccuracyBenchmarkOutOfProcess()
{
IgnoreIfDebug();
// HACK: forcing a reference to System.Configuration
// WAITINGFOR: https://github.com/PerfDotNet/BenchmarkDotNet/issues/234
GC.KeepAlive(typeof(ConfigurationManager));
var config = CreateHighAccuracyConfig(outOfProcess: true);
var runState = SelfTestCompetition.Run <HighAccuracyBenchmarkOutOfProcess>(config);
var messages = runState.GetMessages();
if (messages.All(m => m.MessageText != "CompetitionAnalyser: All competition limits are ok."))
{
Assert.Ignore("The environment does not provide accurate timings. Test results cannot be trusted.");
}
}
public static void CompetitionTooSlowOk()
{
var overrideConfig = CompetitionHelpers
.CreateConfig(typeof(TooSlowBenchmark))
.WithLongRunningBenchmarkLimit(TimeSpan.FromMinutes(2));
var runState = SelfTestCompetition.Run <TooSlowBenchmark>(overrideConfig);
var messages = runState.GetMessages();
var summary = runState.LastRunSummary;
Assert.AreEqual(summary?.ValidationErrors.Length, 0);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 0);
}
public static void CompetitionTooSlowOk()
{
var overrideConfig = new ManualCompetitionConfig(SelfTestConfig)
{
AllowLongRunningBenchmarks = true
};
var runState = SelfTestCompetition.Run <TooSlowBenchmark>(overrideConfig);
var messages = runState.GetMessages();
var summary = runState.LastRunSummary;
Assert.AreEqual(summary?.ValidationErrors.Length, 0);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 0);
}
public static void CompetitionNoBaselineFailBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <NoBaselineFailBenchmark>(SelfTestConfig);
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
AssertCompetitionCompleted(runState, MessageSeverity.SetupError);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.SetupError);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Analyser);
Assert.AreEqual(messages[0].MessageText, "The benchmark NoBaselineFailBenchmark has no baseline.");
}
public static void CompetitionXmlBaselineChangedBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <XmlBaselineChangedBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
AssertCompetitionCompleted(runState, MessageSeverity.SetupError);
Assert.AreEqual(messages.Length, 2);
Assert.AreEqual(
messages[0].MessageText,
"Target Baseline. Baseline flag on the method and in the annotation do not match.");
Assert.AreEqual(
messages[1].MessageText,
"Target SlowerX20. Baseline flag on the method and in the annotation do not match.");
}
public static void CompetitionLimitsEmptyFailBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <LimitsEmptyFailBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount); // 3x rerun
AssertCompetitionCompleted(runState, MessageSeverity.Warning, runNumber: 1);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.Warning);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Analyser);
Assert.AreEqual(
messages[0].MessageText,
"Some benchmark metrics are empty and were ignored. Empty metrics are: SlowerX10: Scaled.");
}
public static void CompetitionBadLimitsBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <BadLimitsBenchmark>();
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
AssertCompetitionCompleted(runState, MessageSeverity.ExecutionError, skipSummary: true);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.ExecutionError);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Runner);
Assert.That(
messages[0].MessageText,
Does.StartWith("Benchmark BadLimitsBenchmark failed. Exception: Invalid range [20.2..5.5]."));
}
public static void CompetitionTooFastBenchmark()
{
var runState = SelfTestCompetition.Run <TooFastBenchmark>(_debugConfig);
var messages = runState.GetMessages();
var summary = runState.LastRunSummary;
Assert.AreEqual(summary?.ValidationErrors.Length, 0);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.Warning);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Analyser);
Assert.AreEqual(
messages[0].MessageText,
"The benchmark(s) TooFast, TooFast2 run faster than 0.0015ms. Results cannot be trusted.");
}
public static void CompetitionTooFastBenchmark()
{
var runState = SelfTestCompetition.Run <TooFastBenchmark>();
var messages = runState.GetMessages();
var summary = runState.LastRunSummary;
Assert.AreEqual(summary?.ValidationErrors.Length, 0);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 3);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.Warning);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Analyser);
Assert.AreEqual(
messages[0].MessageText,
"Benchmarks TooFast, TooFastX5: measured run time is less than 1.00 us. " +
"Timings are imprecise as they are too close to the timer resolution.");
Assert.AreEqual(messages[1].RunNumber, 1);
Assert.AreEqual(messages[1].RunMessageNumber, 2);
Assert.AreEqual(messages[1].MessageSeverity, MessageSeverity.Informational);
Assert.AreEqual(messages[1].MessageSource, MessageSource.Analyser);
Assert.AreEqual(
messages[1].MessageText,
"Target TooFast. Metric validation skipped as the method is not marked with CompetitionBenchmarkAttribute.");
Assert.AreEqual(messages[2].RunNumber, 1);
Assert.AreEqual(messages[2].RunMessageNumber, 3);
Assert.AreEqual(messages[2].MessageSeverity, MessageSeverity.Informational);
Assert.AreEqual(messages[2].MessageSource, MessageSource.Analyser);
Assert.AreEqual(
messages[2].MessageText,
"Target TooFastX5. Metric validation skipped as the method is not marked with CompetitionBenchmarkAttribute.");
}
public static void CompetitionTooSlowBenchmark()
{
var runState = SelfTestCompetition.Run <TooSlowBenchmark>();
var messages = runState.GetMessages();
var summary = runState.LastRunSummary;
Assert.AreEqual(summary?.ValidationErrors.Length, 0);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.Warning);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Analyser);
Assert.AreEqual(
messages[0].MessageText,
"Benchmark TooSlow: run takes more than 500.0 ms." +
" Consider to rewrite the test as peek timings will be hidden by averages.");
}
public static void CompetitionBadLimitsBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
var runState = SelfTestCompetition.Run <BadLimitsBenchmark>(SelfTestConfig);
var messages = runState.GetMessages();
Assert.AreEqual(_callCounter, ExpectedRunCount);
AssertCompetitionCompleted(runState, MessageSeverity.ExecutionError, skipSummary: true);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.ExecutionError);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Runner);
Assert.That(
messages[0].MessageText,
Does.StartWith(
"Benchmark BadLimitsBenchmark failed. Exception: Please check competition limits. " +
"The minRatio (20.2) should not be greater than the maxRatio (5.5)."));
}
public static void CompetitionTooSlowBenchmark()
{
var runState = SelfTestCompetition.Run <TooSlowBenchmark>();
var messages = runState.GetMessages();
var summary = runState.LastRunSummary;
Assert.AreEqual(summary?.ValidationErrors.Length, 0);
Assert.AreEqual(runState.RunNumber, 1);
Assert.AreEqual(runState.RunsLeft, 0);
Assert.AreEqual(runState.RunLimitExceeded, false);
Assert.AreEqual(runState.LooksLikeLastRun, true);
Assert.AreEqual(messages.Length, 1);
Assert.AreEqual(messages[0].RunNumber, 1);
Assert.AreEqual(messages[0].RunMessageNumber, 1);
Assert.AreEqual(messages[0].MessageSeverity, MessageSeverity.Warning);
Assert.AreEqual(messages[0].MessageSource, MessageSource.Analyser);
Assert.AreEqual(
messages[0].MessageText,
"Benchmark TooSlow: measured run time is greater than 500.0 ms. " +
"There's a risk the peak timings were hidden by averages. " +
"Consider to reduce the number of iterations performed per each measurement.");
}
