public void Test()
{
var summary = BenchmarkTestExecutor.CanExecute <BaselineDeltaResultExtenderErrorTest>(fullValidation: false);
// You can't have more than 1 method in a class with [Benchmark(Baseline = true)]
Assert.True(summary.HasCriticalValidationErrors);
}
public void Test()
{
// This is the common way to run benchmarks, it should wire up the BenchmarkBaselineDeltaResultExtender for us
// BenchmarkTestExecutor.CanExecute(..) calls BenchmarkRunner.Run(..) under the hood
var summary = BenchmarkTestExecutor.CanExecute <BaselineDiffColumnTest>();
var table = summary.Table;
var headerRow = table.FullHeader;
var column = summary.Config.GetColumns().OfType <BaselineDiffColumn>().FirstOrDefault();
Assert.NotNull(column);
Assert.Equal(column.ColumnName, headerRow.Last());
var testNameColumn = Array.FindIndex(headerRow, c => c == "Method");
var extraColumn = Array.FindIndex(headerRow, c => c == column.ColumnName);
foreach (var row in table.FullContent)
{
Assert.Equal(row.Length, extraColumn + 1);
if (row[testNameColumn] == "BenchmarkSlow") // This is our baseline
{
Assert.Equal("1.00", row[extraColumn]);
}
else if (row[testNameColumn] == "BenchmarkFast") // This should have been compared to the baseline
{
Assert.Contains(".", row[extraColumn]);
}
}
}
public void Test()
{
var summary = BenchmarkTestExecutor.CanExecute <BaselineDeltaResultExtenderHandlesBenchmarkErrorTest>(fullValidation: false);
var table = summary.Table;
var headerRow = table.FullHeader;
var column = summary.Config.GetColumns().OfType <BaselineDiffColumn>().FirstOrDefault();
Assert.NotNull(column);
Assert.Equal(column.ColumnName, headerRow.Last());
var testNameColumn = Array.FindIndex(headerRow, c => c == "Method");
var extraColumn = Array.FindIndex(headerRow, c => c == column.ColumnName);
foreach (var row in table.FullContent)
{
Assert.Equal(row.Length, extraColumn + 1);
if (row[testNameColumn] == "BenchmarkSlow") // This is our baseline
{
Assert.Equal("1.00", row[extraColumn]);
}
else if (row[testNameColumn] == "BenchmarkThrows") // This should have "?" as it threw an error
{
Assert.Contains("?", row[extraColumn]);
}
}
}
public void Test()
{
var logger = new AccumulationLogger();
var columns = new[]
{
StatisticColumn.StdDev,
StatisticColumn.Min,
StatisticColumn.Q1,
StatisticColumn.Median,
StatisticColumn.Q3,
StatisticColumn.Max,
StatisticColumn.OperationsPerSecond,
StatisticColumn.P85,
StatisticColumn.P95,
StatisticColumn.P95
};
var config = DefaultConfig.Instance.With(logger).With(columns);
var summary = BenchmarkTestExecutor.CanExecute <Target>(config);
output.WriteLine(logger.GetLog());
var table = summary.Table;
var headerRow = table.FullHeader;
foreach (var column in columns)
{
Assert.True(headerRow.Contains(column.ColumnName));
}
}
public void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
BenchmarkTestExecutor.CanExecute <SetupAttributeTest>(config);
Assert.Contains("// ### Setup called ###" + Environment.NewLine, logger.GetLog());
}
public void Test()
{
var testExporter = new TestExporter();
var config = DefaultConfig.Instance.With(testExporter);
BenchmarkTestExecutor.CanExecute <BaselineDeltaResultExtenderNoBaselineTest>(config);
// Ensure that when the TestBenchmarkExporter() was run, it wasn't passed an instance of "BenchmarkBaselineDeltaResultExtender"
Assert.False(testExporter.ExportCalled);
Assert.True(testExporter.ExportToFileCalled);
}
public static void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
BenchmarkTestExecutor.CanExecute <ParamsTestStaticField>(config);
foreach (var param in new[] { 1, 2 })
{
Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog());
}
Assert.DoesNotContain($"// ### New Parameter 0 ###" + Environment.NewLine, logger.GetLog());
}
private void Verify(Platform platform, Type benchmark, string failureText)
{
var logger = new AccumulationLogger();
// make sure we get an output in the TestRunner log
var config = new PlatformConfig(platform).With(logger).With(ConsoleLogger.Default);
BenchmarkTestExecutor.CanExecute(benchmark, config);
var testLog = logger.GetLog();
Assert.DoesNotContain(failureText, testLog);
Assert.DoesNotContain(BenchmarkNotFound, testLog);
}
public void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
BenchmarkTestExecutor.CanExecute <InnerClassTest>(config);
var testLog = logger.GetLog();
Assert.Contains("// ### BenchmarkInnerClass method called ###" + Environment.NewLine, testLog);
Assert.Contains("// ### BenchmarkGenericInnerClass method called ###" + Environment.NewLine, testLog);
Assert.DoesNotContain("No benchmarks found", logger.GetLog());
}
public void Test()
{
// This is the common way to run benchmarks, it should wire up the BenchmarkBaselineDeltaResultExtender for us
var config = DefaultConfig.Instance
.With(BaselineDiffColumn.Scaled50)
.With(BaselineDiffColumn.Scaled85)
.With(BaselineDiffColumn.Scaled95);
var summary = BenchmarkTestExecutor.CanExecute <BaselineScaledColumnsTest>(config);
var table = summary.Table;
var headerRow = table.FullHeader;
var columns = summary.Config.GetColumns().OfType <BaselineDiffColumn>().ToArray();
Assert.Equal(columns.Length, 4);
Assert.Equal(columns[0].ColumnName, headerRow[headerRow.Length - 4]);
Assert.Equal(columns[1].ColumnName, headerRow[headerRow.Length - 3]);
Assert.Equal(columns[2].ColumnName, headerRow[headerRow.Length - 2]);
Assert.Equal(columns[3].ColumnName, headerRow[headerRow.Length - 1]);
var testNameColumn = Array.FindIndex(headerRow, c => c == "Method");
var parseCulture = EnvironmentInfo.MainCultureInfo;
foreach (var row in table.FullContent)
{
Assert.Equal(row.Length, headerRow.Length);
if (row[testNameColumn] == "BenchmarkFast") // This is our baseline
{
Assert.Equal("1.00", row[headerRow.Length - 4]); // Scaled
Assert.Equal("1.00", row[headerRow.Length - 3]); // S50
Assert.Equal("1.00", row[headerRow.Length - 2]); // S85
Assert.Equal("1.00", row[headerRow.Length - 1]); // S95
}
else if (row[testNameColumn] == "BenchmarkSlow") // This should have been compared to the baseline
{
var min = 3.0; // 3.7
var max = 5.0; // 4.3
var scaled = double.Parse(row[headerRow.Length - 4], parseCulture);
Assert.InRange(scaled, min, max);
var s50 = double.Parse(row[headerRow.Length - 3], parseCulture);
Assert.InRange(s50, min, max);
var s85 = double.Parse(row[headerRow.Length - 2], parseCulture);
Assert.InRange(s85, min, max);
var s95 = double.Parse(row[headerRow.Length - 1], parseCulture);
Assert.InRange(s95, min, max);
}
}
}
public void Test()
{
var logger = new OutputLogger(output);
var config = DefaultConfig.Instance.With(logger);
BenchmarkTestExecutor.CanExecute <FlatClassBenchmark>(config);
var expected1 = $"// ### Benchmark: SerializationLibrary1, Type: {typeof(FlatClassBenchmark).Name} ###";
Assert.Contains(expected1, logger.GetLog());
logger.ClearLog();
BenchmarkTestExecutor.CanExecute <DoubleArrayBenchmark>(config);
var expected2 = $"// ### Benchmark: SerializationLibrary2, Type: {typeof(DoubleArrayBenchmark).Name} ###";
Assert.Contains(expected2, logger.GetLog());
}
public void CustomToolchain()
{
var logger = new OutputLogger(output);
var generator = new MyGenerator();
var builder = new MyBuilder();
var executor = new MyExecutor();
var myToolchain = new Toolchain("My", generator, builder, executor);
var job = Job.Default.With(myToolchain).With(Mode.SingleRun).WithLaunchCount(1).WithWarmupCount(1).WithTargetCount(1);
var config = DefaultConfig.Instance.With(job).With(logger);
BenchmarkTestExecutor.CanExecute <ToolchainTest>(config, fullValidation: false);
Assert.True(generator.Done);
Assert.True(builder.Done);
Assert.True(executor.Done);
}
public void Test()
{
logger.ClearLog();
var results = BenchmarkTestExecutor.CanExecute <ModeTests>(new ModeConfig());
Assert.Equal(4, results.Benchmarks.Count());
Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.SingleRun && b.Target.Method.Name == "BenchmarkWithVoid"));
Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.SingleRun && b.Target.Method.Name == "BenchmarkWithReturnValue"));
Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.Throughput && b.Target.Method.Name == "BenchmarkWithVoid"));
Assert.Equal(1, results.Benchmarks.Count(b => b.Job.Mode == Mode.Throughput && b.Target.Method.Name == "BenchmarkWithReturnValue"));
var testLog = logger.GetLog();
Assert.Contains("// ### Benchmark with void called ###", testLog);
Assert.Contains("// ### Benchmark with return value called ###", testLog);
Assert.DoesNotContain("No benchmarks found", logger.GetLog());
}
public void Test()
{
// System.InvalidOperationException : Property "StaticParamProperty" must be public and writable if it has the [Params(..)] attribute applied to it
Assert.Throws <InvalidOperationException>(() => BenchmarkTestExecutor.CanExecute <ParamsTestStaticPrivatePropertyError>());
}
public static void Test()
{
// System.InvalidOperationException : Field "StaticParamField" must be public if it has the [Params(..)] attribute applied to it
Assert.Throws <InvalidOperationException>(() => BenchmarkTestExecutor.CanExecute <ParamsTestStaticPrivateFieldError>());
}
public void AnyValueCanBeReturned()
{
BenchmarkTestExecutor.CanExecute <ValuesReturnedByBenchmarkTest>();
}
public void AreSupported()
{
BenchmarkTestExecutor.CanExecute <CharactersAsParams>();
}
public void Test()
{
Assert.Throws <InvalidOperationException>(() => BenchmarkTestExecutor.CanExecute <SetupAttributeInvalidMethodTest>());
}
public void AreSupported()
{
BenchmarkTestExecutor.CanExecute <NestedEnumsAsParams>();
}