public void RunStrategiesAreSupported() { var logger = new OutputLogger(Output); var config = ManualConfig.CreateEmpty() .With(DefaultColumnProviders.Instance) .With(logger) .With(new Job(Job.Dry) { Run = { RunStrategy = RunStrategy.ColdStart } }) .With(new Job(Job.Dry) { Run = { RunStrategy = RunStrategy.Throughput } }); var results = CanExecute <ModeBenchmarks>(config); Assert.Equal(4, results.Benchmarks.Count()); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Run.RunStrategy == RunStrategy.ColdStart && b.Target.Method.Name == "BenchmarkWithVoid")); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Run.RunStrategy == RunStrategy.ColdStart && b.Target.Method.Name == "BenchmarkWithReturnValue")); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Run.RunStrategy == RunStrategy.Throughput && b.Target.Method.Name == "BenchmarkWithVoid")); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Run.RunStrategy == RunStrategy.Throughput && b.Target.Method.Name == "BenchmarkWithReturnValue")); string testLog = logger.GetLog(); Assert.Contains("// ### Benchmark with void called ###", testLog); Assert.Contains("// ### Benchmark with return value called ###", testLog); Assert.DoesNotContain("No benchmarks found", logger.GetLog()); }
public void AllSetupAndCleanupMethodRunsForSpecificBenchmark() { var logger = new OutputLogger(Output); var miniJob = Job.Default.With(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); var config = CreateSimpleConfig(logger, miniJob); CanExecute <Benchmarks>(config); Output.WriteLine(OutputDelimiter); Output.WriteLine(OutputDelimiter); Output.WriteLine(OutputDelimiter); var firstActualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(FirstPrefix)).ToArray(); foreach (string line in firstActualLogLines) { Output.WriteLine(line); } Assert.Equal(firstExpectedLogLines, firstActualLogLines); var secondActualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(SecondPrefix)).ToArray(); foreach (string line in secondActualLogLines) { Output.WriteLine(line); } Assert.Equal(secondExpectedLogLines, secondActualLogLines); }
public void InProcessBenchmarkAllCasesDiagnoserTest() { var logger = new OutputLogger(Output); var config = new ManualConfig() .With(Job.Default.With(InProcessToolchain.Instance)) .With(MemoryDiagnoser.Default) .With(logger) .With(DefaultColumnProviders.Instance); try { BenchmarkAllCases.Counter = 0; var summary = CanExecute <BenchmarkAllocates>(config); var testLog = logger.GetLog(); Assert.Contains("// Benchmark: BenchmarkAllocates.Allocates:", testLog); Assert.DoesNotContain("No benchmarks found", logger.GetLog()); Assert.True(summary.Reports.Sum(r => r.GcStats.AllocatedBytes) > 0); } finally { BenchmarkAllCases.Counter = 0; } }
public void ModesAreSupported() { var logger = new OutputLogger(Output); var config = ManualConfig.CreateEmpty() .With(DefaultConfig.Instance.GetColumns().ToArray()) .With(logger) .With(Job.Dry.With(Mode.SingleRun)) .With(Job.Dry.With(Mode.Throughput)); var results = CanExecute <ModeBenchmarks>(config); Assert.Equal(4, results.Benchmarks.Count()); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.SingleRun && b.Target.Method.Name == "BenchmarkWithVoid")); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.SingleRun && b.Target.Method.Name == "BenchmarkWithReturnValue")); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.Throughput && b.Target.Method.Name == "BenchmarkWithVoid")); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.Throughput && b.Target.Method.Name == "BenchmarkWithReturnValue")); var testLog = logger.GetLog(); Assert.Contains("// ### Benchmark with void called ###", testLog); Assert.Contains("// ### Benchmark with return value called ###", testLog); Assert.DoesNotContain("No benchmarks found", logger.GetLog()); }
public void ParamsSupportPropertyWithPublicSetter() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute<ParamsTestProperty>(config); foreach (var param in new[] { 1, 2 }) Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog()); Assert.DoesNotContain($"// ### New Parameter {default(int)} ###" + Environment.NewLine, logger.GetLog()); }
public void Test() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute<ParamsTestStaticField>(config); foreach (var param in new[] { 1, 2 }) Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog()); Assert.DoesNotContain($"// ### New Parameter 0 ###" + Environment.NewLine, logger.GetLog()); }
public void InnerClassesAreSupported() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute <Inner>(config); var testLog = logger.GetLog(); Assert.Contains("// ### BenchmarkInnerClass method called ###" + Environment.NewLine, testLog); Assert.Contains("// ### BenchmarkGenericInnerClass method called ###" + Environment.NewLine, testLog); Assert.DoesNotContain("No benchmarks found", logger.GetLog()); }
public void InProcessBenchmarkEmitsSameIL(Type benchmarkType) { var logger = new OutputLogger(Output); var config = CreateInProcessAndRoslynConfig(logger); var summary = CanExecute(benchmarkType, config); DiffEmit(summary); string testLog = logger.GetLog(); Assert.Contains(benchmarkType.Name, testLog); Assert.DoesNotContain("No benchmarks found", logger.GetLog()); }
public void GenericClassesAreSupported() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute<FlatClassBenchmark>(config); var expected1 = $"// ### Benchmark: SerializationLibrary1, Type: {typeof(FlatClassBenchmark).Name} ###"; Assert.Contains(expected1, logger.GetLog()); logger.ClearLog(); CanExecute<DoubleArrayBenchmark>(config); var expected2 = $"// ### Benchmark: SerializationLibrary2, Type: {typeof(DoubleArrayBenchmark).Name} ###"; Assert.Contains(expected2, logger.GetLog()); }
public void Test() { var logger = new OutputLogger(output); var config = DefaultConfig.Instance.With(logger); var summary = BenchmarkRunner.Run<PerformanceUnitTest>(config); // Sanity checks, to be sure that the different benchmarks actually run var testOutput = logger.GetLog(); Assert.Contains("// ### Slow Benchmark called ###" + Environment.NewLine, testOutput); Assert.Contains("// ### Fast Benchmark called ###" + Environment.NewLine, testOutput); // Check that slow benchmark is actually slower than the fast benchmark! var slowBenchmarkRun = summary.GetRunsFor<PerformanceUnitTest>(r => r.SlowBenchmark()).First(); var fastBenchmarkRun = summary.GetRunsFor<PerformanceUnitTest>(r => r.FastBenchmark()).First(); Assert.True(slowBenchmarkRun.GetAverageNanoseconds() > fastBenchmarkRun.GetAverageNanoseconds(), string.Format("Expected SlowBenchmark: {0:N2} ns to be MORE than FastBenchmark: {1:N2} ns", slowBenchmarkRun.GetAverageNanoseconds(), fastBenchmarkRun.GetAverageNanoseconds())); Assert.True(slowBenchmarkRun.GetOpsPerSecond() < fastBenchmarkRun.GetOpsPerSecond(), string.Format("Expected SlowBenchmark: {0:N2} Ops to be LESS than FastBenchmark: {1:N2} Ops", slowBenchmarkRun.GetOpsPerSecond(), fastBenchmarkRun.GetOpsPerSecond())); // Whilst we're at it, let's do more specific Asserts as we know what the elasped time should be var slowBenchmarkReport = summary.GetReportFor<PerformanceUnitTest>(r => r.SlowBenchmark()); var fastBenchmarkReport = summary.GetReportFor<PerformanceUnitTest>(r => r.FastBenchmark()); foreach (var slowRun in slowBenchmarkReport.GetResultRuns()) Assert.InRange(slowRun.GetAverageNanoseconds() / 1000.0 / 1000.0, low: 98, high: 102); foreach (var fastRun in fastBenchmarkReport.GetResultRuns()) Assert.InRange(fastRun.GetAverageNanoseconds() / 1000.0 / 1000.0, low: 14, high: 17); }
public void GlobalCleanupTargetSpecificMethodTest() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute <GlobalCleanupAttributeTargetBenchmarks>(config); string log = logger.GetLog(); Assert.Contains(BaselineBenchmarkCalled + Environment.NewLine, log); Assert.True( log.IndexOf(BaselineBenchmarkCalled + Environment.NewLine) < log.IndexOf(BaselineGlobalCleanupCalled + Environment.NewLine)); Assert.Contains(FirstGlobalCleanupCalled + Environment.NewLine, log); Assert.Contains(FirstBenchmarkCalled + Environment.NewLine, log); Assert.True( log.IndexOf(FirstBenchmarkCalled + Environment.NewLine) < log.IndexOf(FirstGlobalCleanupCalled + Environment.NewLine)); Assert.Contains(SecondGlobalCleanupCalled + Environment.NewLine, log); Assert.Contains(SecondBenchmarkCalled + Environment.NewLine, log); Assert.True( log.IndexOf(SecondBenchmarkCalled + Environment.NewLine) < log.IndexOf(SecondGlobalCleanupCalled + Environment.NewLine)); }
public void Test() { var logger = new OutputLogger(output); var config = DefaultConfig.Instance.With(logger); BenchmarkRunner.Run <FlatClassBenchmark>(config); var expected1 = $"// ### Benchmark: SerializationLibrary1, Type: {typeof(FlatClassBenchmark).Name} ###"; Assert.Contains(expected1, logger.GetLog()); logger.ClearLog(); BenchmarkRunner.Run <DoubleArrayBenchmark>(config); var expected2 = $"// ### Benchmark: SerializationLibrary2, Type: {typeof(DoubleArrayBenchmark).Name} ###"; Assert.Contains(expected2, logger.GetLog()); }
private void Verify(Runtime runtime, Jit jit, Platform platform, string expectedText) { var logger = new OutputLogger(Output); var config = new PlatformConfig(runtime, jit, platform).With(logger).With(DefaultColumnProviders.Instance); BenchmarkRunner.Run(new[] { BenchmarkConverter.TypeToBenchmarks(typeof(TestBenchmark), config) }); Assert.Contains(expectedText, logger.GetLog()); }
public void ExceptionMessageIsNotLost() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute <ThrowingException>(config, fullValidation: false); Assert.Contains(ErrorMessage, logger.GetLog()); }
private void Verify(Platform platform, Type benchmark, string failureText) { var logger = new OutputLogger(Output); // make sure we get an output in the TestRunner log var config = new PlatformConfig(platform).With(logger).With(DefaultColumnProviders.Instance); CanExecute(benchmark, config); var testLog = logger.GetLog(); Assert.DoesNotContain(failureText, testLog); Assert.DoesNotContain(BenchmarkNotFound, testLog); }
public void ColdStart() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute <ColdStartBench>(config); string log = logger.GetLog(); Assert.Contains($"{CounterPrefix}1", log); Assert.DoesNotContain($"{CounterPrefix}2", log); }
public void CleanupMethodRunsTest() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute<CleanupAttributeBenchmarks>(config); string log = logger.GetLog(); Assert.Contains(CleanupCalled + Environment.NewLine, log); Assert.True( log.IndexOf(CleanupCalled + Environment.NewLine) > log.IndexOf(BenchmarkCalled + Environment.NewLine)); }
private void Verify(Platform platform, Type benchmark, string failureText) { var logger = new OutputLogger(Output); // make sure we get an output in the TestRunner log var config = new PlatformConfig(platform) .With(logger) .With(DefaultConfig.Instance.GetColumns().ToArray()); CanExecute(benchmark, config); var testLog = logger.GetLog(); Assert.DoesNotContain(failureText, testLog); Assert.DoesNotContain(BenchmarkNotFound, testLog); }
public void GlobalCleanupMethodRunsTest() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute <GlobalCleanupAttributeBenchmarks>(config); string log = logger.GetLog(); Assert.Contains(GlobalCleanupCalled + System.Environment.NewLine, log); Assert.True( log.IndexOf(GlobalCleanupCalled + System.Environment.NewLine) > log.IndexOf(BenchmarkCalled + System.Environment.NewLine)); }
private void Verify(Platform platform, Type benchmark, string failureText) { var logger = new OutputLogger(Output); var config = ManualConfig.CreateEmpty() .With(Job.Dry.With(platform)) .With(logger); // make sure we get an output in the TestRunner log CanExecute(benchmark, config); var testLog = logger.GetLog(); Assert.DoesNotContain(failureText, testLog); Assert.DoesNotContain(BenchmarkNotFound, testLog); }
public void InProcessBenchmarkAllCasesDelegateCombineSupported() { var logger = new OutputLogger(Output); var config = CreateInProcessConfig(BenchmarkActionCodegen.DelegateCombine, logger); try { BenchmarkAllCases.Counter = 0; var summary = CanExecute <BenchmarkAllCases>(config); var testLog = logger.GetLog(); Assert.Contains("// Benchmark: BenchmarkAllCases.InvokeOnceVoid:", testLog); Assert.DoesNotContain("No benchmarks found", logger.GetLog()); // Operations + Setup + Cleanup var expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2); Assert.Equal(expectedCount, BenchmarkAllCases.Counter); } finally { BenchmarkAllCases.Counter = 0; } }
public void InProcessBenchmarkSimpleCasesReflectionEmitSupported() { var logger = new OutputLogger(Output); var config = CreateInProcessConfig(logger); try { BenchmarkAllCases.Counter = 0; var summary = CanExecute <BenchmarkAllCases>(config); string testLog = logger.GetLog(); Assert.Contains("// Benchmark: BenchmarkAllCases.InvokeOnceVoid:", testLog); Assert.DoesNotContain("No benchmarks found", logger.GetLog()); // Operations + GlobalSetup + GlobalCleanup long expectedCount = summary.Reports.SelectMany(r => r.AllMeasurements).Sum(m => m.Operations + 2); Assert.Equal(expectedCount, BenchmarkAllCases.Counter); } finally { BenchmarkAllCases.Counter = 0; } }
public void ModesAreSupported() { var logger = new OutputLogger(Output); var config = ManualConfig.CreateEmpty() .With(DefaultConfig.Instance.GetColumns().ToArray()) .With(logger) .With(Job.Dry.With(Mode.SingleRun)) .With(Job.Dry.With(Mode.Throughput)); var results = CanExecute<ModeBenchmarks>(config); Assert.Equal(4, results.Benchmarks.Count()); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.SingleRun && b.Target.Method.Name == "BenchmarkWithVoid")); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.SingleRun && b.Target.Method.Name == "BenchmarkWithReturnValue")); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.Throughput && b.Target.Method.Name == "BenchmarkWithVoid")); Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.Throughput && b.Target.Method.Name == "BenchmarkWithReturnValue")); var testLog = logger.GetLog(); Assert.Contains("// ### Benchmark with void called ###", testLog); Assert.Contains("// ### Benchmark with return value called ###", testLog); Assert.DoesNotContain("No benchmarks found", logger.GetLog()); }
public void AllSetupAndCleanupMethodRunsAsyncTest() { var logger = new OutputLogger(Output); var miniJob = Job.Default.WithStrategy(RunStrategy.Monitoring).WithWarmupCount(2).WithIterationCount(3).WithInvocationCount(1).WithUnrollFactor(1).WithId("MiniJob"); var config = CreateSimpleConfig(logger, miniJob); CanExecute <AllSetupAndCleanupAttributeBenchmarksAsync>(config); var actualLogLines = logger.GetLog().Split('\r', '\n').Where(line => line.StartsWith(Prefix)).ToArray(); foreach (string line in actualLogLines) { Output.WriteLine(line); } Assert.Equal(expectedLogLines, actualLogLines); }
public void Test() { var logger = new OutputLogger(output); var config = DefaultConfig.Instance.With(logger); var summary = BenchmarkRunner.Run <PerformanceUnitTest>(config); // Sanity checks, to be sure that the different benchmarks actually run var testOutput = logger.GetLog(); Assert.Contains("// ### Slow Benchmark called ###" + Environment.NewLine, testOutput); Assert.Contains("// ### Fast Benchmark called ###" + Environment.NewLine, testOutput); // Check that slow benchmark is actually slower than the fast benchmark! var slowBenchmarkRun = summary.GetRunsFor <PerformanceUnitTest>(r => r.SlowBenchmark()).First(); var fastBenchmarkRun = summary.GetRunsFor <PerformanceUnitTest>(r => r.FastBenchmark()).First(); Assert.True(slowBenchmarkRun.GetAverageNanoseconds() > fastBenchmarkRun.GetAverageNanoseconds(), string.Format("Expected SlowBenchmark: {0:N2} ns to be MORE than FastBenchmark: {1:N2} ns", slowBenchmarkRun.GetAverageNanoseconds(), fastBenchmarkRun.GetAverageNanoseconds())); Assert.True(slowBenchmarkRun.GetOpsPerSecond() < fastBenchmarkRun.GetOpsPerSecond(), string.Format("Expected SlowBenchmark: {0:N2} Ops to be LESS than FastBenchmark: {1:N2} Ops", slowBenchmarkRun.GetOpsPerSecond(), fastBenchmarkRun.GetOpsPerSecond())); // Whilst we're at it, let's do more specific Asserts as we know what the elasped time should be var slowBenchmarkReport = summary.GetReportFor <PerformanceUnitTest>(r => r.SlowBenchmark()); var fastBenchmarkReport = summary.GetReportFor <PerformanceUnitTest>(r => r.FastBenchmark()); foreach (var slowRun in slowBenchmarkReport.GetResultRuns()) { Assert.InRange(slowRun.GetAverageNanoseconds() / 1000.0 / 1000.0, low: 98, high: 102); } foreach (var fastRun in fastBenchmarkReport.GetResultRuns()) { Assert.InRange(fastRun.GetAverageNanoseconds() / 1000.0 / 1000.0, low: 14, high: 17); } }
public void ParamsSupportFSharpEnums() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute <EnumParamsTest>(config); foreach (var param in new[] { TestEnum.A, TestEnum.B }) { Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog()); } Assert.DoesNotContain($"// ### New Parameter {TestEnum.C} ###" + Environment.NewLine, logger.GetLog()); }
public void ParamsSupportPropertyWithPublicSetter() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute <ParamsTestProperty>(config); foreach (var param in new[] { 1, 2 }) { Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog()); } Assert.DoesNotContain($"// ### New Parameter {default(int)} ###" + Environment.NewLine, logger.GetLog()); }
public void Test() { var logger = new OutputLogger(Output); var config = CreateSimpleConfig(logger); CanExecute <ParamsTestStaticField>(config); foreach (var param in new[] { 1, 2 }) { Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog()); } Assert.DoesNotContain($"// ### New Parameter 0 ###" + Environment.NewLine, logger.GetLog()); }