public static void TestInProcessBenchmarkStandardEngineUnroll()
{
ResetCounters();
var config = CompetitionHelpers.ConfigForAssembly
.WithModifier(
new Job
{
Infrastructure =
{
EngineFactory = new EngineFactory()
},
Run =
{
InvocationCount = 20,
UnrollFactor = 5
}
})
.WithModifier(
new CompetitionOptions()
{
Checks = { TooFastBenchmarkLimit = TimeSpan.Zero }
});
var summary = SelfTestCompetition
.Run <InProcessBenchmarkAllCases>(config)
.LastRunSummary;
Assert.AreEqual(_callCounter, GetExpectedCount(config, MethodsCount));
Assert.AreEqual(_afterSetupCounter, GetExpectedCount(config, 1));
Assert.AreEqual(_setupCounter, MethodsCount);
Assert.AreEqual(_cleanupCounter, MethodsCount);
Assert.IsFalse(summary?.ValidationErrors.Any());
}
public static void TestInProcessBenchmarkWithValidation()
{
ResetCounters();
// DONTTOUCH: config SHOULD NOT match the default platform (x64).
var config = new ManualCompetitionConfig(
CompetitionHelpers.CreateConfig(
typeof(InProcessBenchmarkAllCases),
new CompetitionFeatures
{
Platform = Platform.X86
}));
config.Add(InProcessValidator.FailOnError);
var summary = SelfTestCompetition
.Run <InProcessBenchmarkAllCases>(config)
.LastRunSummary;
Assert.AreEqual(_callCounter, 0);
Assert.AreEqual(_afterSetupCounter, 0);
Assert.AreEqual(_setupCounter, 0);
Assert.AreEqual(_cleanupCounter, 0);
Assert.AreEqual(summary?.ValidationErrors.Length, 1);
Assert.IsTrue(summary?.ValidationErrors[0].IsCritical);
Assert.AreEqual(
summary?.ValidationErrors[0].Message,
"Job SelfTestConfigX86, EnvMode.Platform was run as X64 (X86 expected). Fix your test runner options.");
}
public static void TestInProcessBenchmarkBurstMode()
{
ResetCounters();
var config = CompetitionHelpers.ConfigForAssembly
.WithModifier(
new Job
{
Infrastructure =
{
#pragma warning disable 618
// TODO: remove pragma after BurstModeEngine will be fixed
EngineFactory = BurstModeEngineFactory.Instance
#pragma warning restore 618
}
})
.WithModifier(
new CompetitionOptions()
{
Checks = { TooFastBenchmarkLimit = TimeSpan.Zero }
});
var summary = SelfTestCompetition
.Run <InProcessBenchmarkAllCases>(config)
.LastRunSummary;
Assert.AreEqual(_callCounter, GetExpectedCount(config, MethodsCount));
Assert.AreEqual(_afterSetupCounter, GetExpectedCount(config, 1));
Assert.AreEqual(_setupCounter, MethodsCount);
Assert.AreEqual(_cleanupCounter, MethodsCount);
Assert.IsFalse(summary?.ValidationErrors.Any());
}
public static void TestInProcessBenchmark()
{
Interlocked.Exchange(ref _callCounter, 0);
Interlocked.Exchange(ref _afterSetupCounter, 0);
var summary = SelfTestCompetition
.Run <InProcessBenchmark>(SelfTestConfig)
.LastRunSummary;
Assert.AreEqual(_callCounter, ExpectedSelfTestRunCount);
Assert.AreEqual(_afterSetupCounter, 2);
Assert.IsFalse(summary?.ValidationErrors.Any());
}
public static void TestInProcessBenchmarkStandardEngine()
{
ResetCounters();
var invocationCount = 1;
var config = CompetitionHelpers.ConfigForAssembly
.WithModifier(
new Job
{
Infrastructure =
{
EngineFactory = new EngineFactory()
}
})
.WithModifier(
new CompetitionOptions()
{
Checks = { TooFastBenchmarkLimit = TimeSpan.Zero }
});
var summary = SelfTestCompetition
.Run <InProcessBenchmarkAllCases>(config)
.LastRunSummary;
// TODO: uncomment after https://github.com/dotnet/BenchmarkDotNet/issues/464
//var iterationsNoUnroll = GetExpectedCountIgnoreUnroll(config, 1);
Assert.AreEqual(_callCounter, GetExpectedCount(config, MethodsCount));
Assert.AreEqual(_afterGlobalSetupCounter, GetExpectedCount(config, 1));
Assert.AreEqual(_afterIterationSetupCounter, invocationCount);
Assert.AreEqual(_globalSetupCounter, MethodsCount - 1);
Assert.AreEqual(_globalSetupTargetCounter, 1);
//Assert.AreEqual(_iterationSetupCounter, MethodsCount * iterationsNoUnroll);
Assert.AreEqual(_globalCleanupCounter, MethodsCount);
//Assert.AreEqual(_iterationCleanupCounter, (MethodsCount - 1) * iterationsNoUnroll);
//Assert.AreEqual(_iterationCleanupTargetCounter, 1 * iterationsNoUnroll);
var a = summary?.ValidationErrors.ToArray();
foreach (var validationError in a)
{
Console.WriteLine(validationError.Message);
}
Assert.IsFalse(a.Any());
}
public static void TestInProcessBenchmarkWithValidation()
{
// DONTTOUCH: config SHOULD NOT match the default platform (x64).
var config = CreateSelfTestConfig(Platform.X86);
config.Add(InProcessValidator.FailOnError);
Interlocked.Exchange(ref _callCounter, 0);
Interlocked.Exchange(ref _afterSetupCounter, 0);
var summary = SelfTestCompetition
.Run <InProcessBenchmark>(config)
.LastRunSummary;
Assert.AreEqual(_callCounter, 0);
Assert.AreEqual(_afterSetupCounter, 0);
Assert.AreEqual(summary?.ValidationErrors.Length, 1);
Assert.IsTrue(summary?.ValidationErrors[0].IsCritical);
Assert.That(summary?.ValidationErrors[0].Message, Does.Contain(", property Platform:"));
}