// TODO: Union this with MockFactory
private Summary CreateSummary(int[] values)
{
var logger = new AccumulationLogger();
var benchmarks = CreateBenchmarks(DefaultConfig.Instance).ToList();
var benchmarkReports = new List <BenchmarkReport>();
for (var x = 0; x < benchmarks.Count; x++)
{
var benchmark = benchmarks[x];
benchmarkReports.Add(CreateReport(benchmark, values[x]));
}
var summary = new Summary(
"MockSummary",
benchmarkReports,
new HostEnvironmentInfoBuilder().Build(),
DefaultConfig.Instance,
"",
TimeSpan.FromMinutes(1),
Array.Empty <ValidationError>());
MarkdownExporter.Default.ExportToLog(summary, logger);
output.WriteLine(logger.GetLog());
return(summary);
}
// TODO: Union this with MockFactory
private Summary CreateSummary(int[] values, RatioStyle ratioStyle, int noise)
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.WithSummaryStyle(SummaryStyle.Default.WithRatioStyle(ratioStyle: ratioStyle));
var benchmarks = CreateBenchmarks(config).ToList();
var benchmarkReports = new List <BenchmarkReport>();
for (var x = 0; x < benchmarks.Count; x++)
{
var benchmark = benchmarks[x];
benchmarkReports.Add(CreateReport(benchmark, values[x], noise));
}
var summary = new Summary(
"MockSummary",
benchmarkReports.ToImmutableArray(),
new HostEnvironmentInfoBuilder().Build(),
string.Empty,
string.Empty,
TimeSpan.FromMinutes(1),
TestCultureInfo.Instance,
ImmutableArray <ValidationError> .Empty);
MarkdownExporter.Default.ExportToLog(summary, logger);
output.WriteLine(logger.GetLog());
return(summary);
}
public void Test()
{
var logger = new AccumulationLogger();
var columns = new[]
{
StatisticColumn.StdDev,
StatisticColumn.Min,
StatisticColumn.Q1,
StatisticColumn.Median,
StatisticColumn.Q3,
StatisticColumn.Max,
StatisticColumn.OperationsPerSecond,
StatisticColumn.P85,
StatisticColumn.P95,
StatisticColumn.P95
};
var config = DefaultConfig.Instance.With(logger).With(columns);
var summary = BenchmarkTestExecutor.CanExecute <Target>(config);
output.WriteLine(logger.GetLog());
var table = summary.Table;
var headerRow = table.FullHeader;
foreach (var column in columns)
{
Assert.True(headerRow.Contains(column.ColumnName));
}
}
public void TestFilteringLoggerImportantArea()
{
var lines = LogInput.Split(new[] { "\r\n" }, StringSplitOptions.None);
var output = new AccumulationLogger();
var logger = new FilteringLogger(output, FilteringLoggerMode.AllMessages);
var config = new ManualConfig();
config.Add(logger);
using (LoggerHelpers.BeginImportantLogScope(config))
{
foreach (var line in lines)
{
logger.WriteLine(line);
}
Assert.AreEqual(output.GetLog(), LogInput + "\r\n");
output = new AccumulationLogger();
logger = new FilteringLogger(output, FilteringLoggerMode.AllMessages);
var i = 0;
foreach (var line in lines)
{
if (i++ % 2 == 0)
{
logger.WriteLineError(line);
}
else
{
logger.WriteLineHint(line);
}
}
Assert.AreEqual(output.GetLog(), LogInput + "\r\n");
output = new AccumulationLogger();
logger = new FilteringLogger(output, FilteringLoggerMode.AllMessages);
LogMessages(logger);
// ReSharper disable once StringLiteralTypo
const string Expected = @"ABCDEFGH
TST0
// !TST1
TST2
// !<--
TST3
// !<--
TST4
TST5
// !-->
TST6
// !-->
TST7
TST8
// !TST9
TST10
";
Assert.AreEqual(output.GetLog(), Expected);
}
}
async void Button_Clicked(object sender, EventArgs e)
{
SetIsRunning(true);
try
{
var logger = new AccumulationLogger();
await Task.Run(() =>
{
var config = default(IConfig);
#if DEBUG
config = new DebugInProcessConfig();
#endif
var summary = BenchmarkRunner.Run <IntroBasic>(config);
MarkdownExporter.Console.ExportToLog(summary, logger);
ConclusionHelper.Print(logger,
summary.BenchmarksCases
.SelectMany(benchmark => benchmark.Config.GetCompositeAnalyser().Analyse(summary))
.Distinct()
.ToList());
});
SetSummary(logger.GetLog());
}
catch (Exception exc)
{
await DisplayAlert("Error", exc.Message, "Ok");
}
finally
{
SetIsRunning(false);
}
}
public void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
BenchmarkTestExecutor.CanExecute <SetupAttributeTest>(config);
Assert.Contains("// ### Setup called ###" + Environment.NewLine, logger.GetLog());
}
public void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
// System.InvalidOperationException : Property "StaticParamProperty" must be public and writable if it has the [Params(..)] attribute applied to it
Assert.Throws <InvalidOperationException>(() => BenchmarkRunner.Run(this.GetType(), config));
}
public static void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
// System.InvalidOperationException : Field "StaticParamField" must be public if it has the [Params(..)] attribute applied to it
Assert.Throws <InvalidOperationException>(() => BenchmarkRunner.Run <ParamsTestStaticPrivateField>(config));
}
// TODO: Union this with MockFactory
private Summary CreateSummary(bool hugeSd, Metric[] metrics)
{
var logger = new AccumulationLogger();
var summary = MockFactory.CreateSummary <MockBenchmarkClass>(DefaultConfig.Instance, hugeSd, metrics);
MarkdownExporter.Default.ExportToLog(summary, logger);
output.WriteLine(logger.GetLog());
return(summary);
}
private Summary CreateSummary(IConfig config)
{
var logger = new AccumulationLogger();
var summary = MockFactory.CreateSummary(config);
MarkdownExporter.Default.ExportToLog(summary, logger);
output.WriteLine(logger.GetLog());
return(summary);
}
private void SourceExceptionMessageIsDisplayed <TBenchmark>(Job job)
{
var logger = new AccumulationLogger();
var config = ManualConfig.CreateEmpty().With(job).With(logger);
CanExecute <TBenchmark>(config, fullValidation: false); // we don't validate here because the report is expected to have no results
Assert.Contains(BenchmarkExceptionMessage, logger.GetLog());
}
private static AccumulationLogger InitAccumulationLogger()
{
var logger = new AccumulationLogger();
logger.WriteLine();
logger.WriteLine();
logger.WriteLine(new string('=', 40));
logger.WriteLine();
return(logger);
}
private SummaryTable CreateTable()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance;
var summary = MockFactory.CreateSummary(config);
var table = new SummaryTable(summary);
MarkdownExporter.Default.ExportToLog(summary, logger);
output.WriteLine(logger.GetLog());
return(table);
}
private void Verify(Platform platform, Type benchmark, string failureText)
{
var logger = new AccumulationLogger();
var config = new PlatformConfig(platform).With(logger);
BenchmarkRunner.Run(benchmark, config);
var testLog = logger.GetLog();
Assert.DoesNotContain(failureText, testLog);
Assert.DoesNotContain(BenchmarkNotFound, testLog);
}
private static void DumpOutputSummaryAtTop(Summary summary, AccumulationLogger logger)
{
if (summary != null)
{
// Dumping the benchmark results to console
MarkdownExporter.Default.ExportToLog(summary, ConsoleLogger.Default);
}
// Dumping all captured output below the benchmark results
ConsoleLogger.Default.WriteLine(logger.GetLog());
}
public void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
BenchmarkTestExecutor.CanExecute <ParamsTestProperty>(config);
foreach (var param in new[] { 1, 2 })
{
Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog());
}
Assert.DoesNotContain($"// ### New Parameter {default(int)} ###" + Environment.NewLine, logger.GetLog());
}
private void Verify(Platform platform, Type benchmark, string failureText)
{
var logger = new AccumulationLogger();
// make sure we get an output in the TestRunner log
var config = new PlatformConfig(platform).With(logger).With(ConsoleLogger.Default);
BenchmarkTestExecutor.CanExecute(benchmark, config);
var testLog = logger.GetLog();
Assert.DoesNotContain(failureText, testLog);
Assert.DoesNotContain(BenchmarkNotFound, testLog);
}
public void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
BenchmarkRunner.Run <InnerClassTest>(config);
var testLog = logger.GetLog();
Assert.Contains("// ### BenchmarkInnerClass method called ###" + Environment.NewLine, testLog);
Assert.Contains("// ### BenchmarkGenericInnerClass method called ###" + Environment.NewLine, testLog);
Assert.DoesNotContain("No benchmarks found", logger.GetLog());
}
public static void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
BenchmarkRunner.Run <ParamsTestStaticField>(config);
foreach (var param in new[] { 1, 2 })
{
Assert.Contains($"// ### New Parameter {param} ###" + Environment.NewLine, logger.GetLog());
}
Assert.DoesNotContain($"// ### New Parameter 0 ###" + Environment.NewLine, logger.GetLog());
}
public void Test()
{
var logger = new AccumulationLogger();
var testExporter = new TestExporter();
var config = DefaultConfig.Instance.With(logger).With(testExporter);
BenchmarkRunner.Run(this.GetType(), config);
// Ensure that when the TestBenchmarkExporter() was run, it wasn't passed an instance of "BenchmarkBaselineDeltaResultExtender"
Assert.False(testExporter.ExportCalled);
Assert.True(testExporter.ExportToFileCalled);
}
public void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
// Run our F# test that lives in "BenchmarkDotNet.IntegrationTests.FSharp"
BenchmarkRunner.Run <FSharpBenchmark.Db>(config);
var testLog = logger.GetLog();
Assert.Contains("// ### F# Benchmark method called ###", testLog);
Assert.DoesNotContain("No benchmarks found", testLog); // TODO: move message to const for all of the test
}
public void Test()
{
var logger = new AccumulationLogger();
var sourceDiagnoser = new SourceDiagnoser();
var config = DefaultConfig.Instance.With(logger).With(sourceDiagnoser);
BenchmarkRunner.Run <SourceDiagnoserTest>(config);
var testOutput = logger.GetLog();
Assert.Contains($"Printing Code for Method: {this.GetType().FullName}.DictionaryEnumeration()", testOutput);
Assert.Contains("PrintAssembly=True", testOutput);
}
// TODO: Union this with MockFactory
private Summary CreateSummary(bool hugeSd, Metric[] metrics)
{
var logger = new AccumulationLogger();
var summary = new Summary(
"MockSummary",
CreateBenchmarks(DefaultConfig.Instance).Select(b => CreateReport(b, hugeSd, metrics)).ToImmutableArray(),
new HostEnvironmentInfoBuilder().Build(),
string.Empty,
string.Empty,
TimeSpan.FromMinutes(1),
ImmutableArray<ValidationError>.Empty);
MarkdownExporter.Default.ExportToLog(summary, logger);
output.WriteLine(logger.GetLog());
return summary;
}
public void TestFilteringLoggerPrefixedOrErrors()
{
var lines = LogInput.Split(new[] { "\r\n" }, StringSplitOptions.None);
var output = new AccumulationLogger();
var logger = new FilteringLogger(output, LogFilter.PrefixedOrErrors);
foreach (var line in lines)
{
logger.WriteLine(line);
}
Assert.AreEqual(output.GetLog(), PrefixedLogOutput);
output = new AccumulationLogger();
logger = new FilteringLogger(output, LogFilter.PrefixedOrErrors);
var i = 0;
foreach (var line in lines)
{
if (i++ % 2 == 0)
{
logger.WriteLineError(line);
}
else
{
logger.WriteLineHint(line);
}
}
Assert.AreEqual(output.GetLog(), LogInput + "\r\n");
output = new AccumulationLogger();
logger = new FilteringLogger(output, LogFilter.PrefixedOrErrors);
LogMessages(logger);
const string Expected = @"AH
TST0
// !TST1
// !<--
TST3
// !<--
TST4
TST5
// !-->
TST6
// !-->
";
Assert.AreEqual(output.GetLog(), Expected);
}
// TODO: Union this with MockFactory
private Summary CreateSummary(bool hugeSd)
{
var logger = new AccumulationLogger();
var summary = new Summary(
"MockSummary",
CreateBenchmarks(DefaultConfig.Instance).Select(b => CreateReport(b, hugeSd)).ToArray(),
MockFactory.MockHostEnvironmentInfo.Default,
DefaultConfig.Instance,
"",
TimeSpan.FromMinutes(1),
Array.Empty <ValidationError>());
MarkdownExporter.Default.ExportToLog(summary, logger);
output.WriteLine(logger.GetLog());
return(summary);
}
public void Test()
{
var logger = new AccumulationLogger();
var config = DefaultConfig.Instance.With(logger);
BenchmarkRunner.Run <ModeTests>(config);
var testLog = logger.GetLog();
Assert.Contains("// ### BenchmarkSingleRunVoid method called ###", testLog);
Assert.Contains("// ### BenchmarkSingleRunWithReturnValue method called ###", testLog);
Assert.Contains("// ### BenchmarkSingleRunVoid method called ###", testLog);
Assert.Contains("// ### BenchmarkSingleRunWithReturnValue method called ###", testLog);
Assert.DoesNotContain("No benchmarks found", logger.GetLog());
}
public void Exporters(CultureInfo cultureInfo)
{
NamerFactory.AdditionalInformation = $"{GetName(cultureInfo)}";
Thread.CurrentThread.CurrentCulture = cultureInfo;
var logger = new AccumulationLogger();
var exporters = GetExporters();
foreach (var exporter in exporters)
{
PrintTitle(logger, exporter);
exporter.ExportToLog(MockFactory.CreateSummary(config), logger);
}
Approvals.Verify(logger.GetLog());
}
public void TestHostLoggerAllMessages()
{
var lines = LogInput.Split(new[] { "\r\n" }, StringSplitOptions.None);
var output = new AccumulationLogger();
var logger = new HostLogger(output, HostLogMode.AllMessages);
foreach (var line in lines)
{
logger.WriteLine(line);
}
Assert.AreEqual(output.GetLog(), LogInput + "\r\n");
output = new AccumulationLogger();
logger = new HostLogger(output, HostLogMode.AllMessages);
foreach (var line in lines)
{
logger.WriteLineError(line);
}
Assert.AreEqual(output.GetLog(), LogInput + "\r\n");
output = new AccumulationLogger();
logger = new HostLogger(output, HostLogMode.AllMessages);
LogMessages(logger);
// ReSharper disable once StringLiteralTypo
const string Expected = @"ABCDEFGH
TST0
// !TST1
TST2
// !<--
TST3
// !<--
TST4
TST5
// !-->
TST6
// !-->
TST7
TST8
// !TST9
TST10
";
Assert.AreEqual(output.GetLog(), Expected);
}
public void Exporters(string cultureInfoName)
{
var cultureInfo = CultureInfos[cultureInfoName];
NamerFactory.AdditionalInformation = $"{GetName(cultureInfo)}";
Thread.CurrentThread.CurrentCulture = cultureInfo;
var logger = new AccumulationLogger();
var exporters = GetExporters();
foreach (var exporter in exporters)
{
PrintTitle(logger, exporter);
exporter.ExportToLog(MockFactory.CreateSummary(config.WithCultureInfo(cultureInfo)), logger);
}
Approvals.Verify(logger.GetLog(), x => x.Replace("\r", string.Empty));
}
public void Test()
{
var logger = new AccumulationLogger();
var gcDiagnoser = new GCDiagnoser();
var config = DefaultConfig.Instance.With(logger).With(gcDiagnoser);
var benchmarks = BenchmarkConverter.TypeToBenchmarks(typeof(GCDiagnoserTests), config);
var summary = BenchmarkRunner.Run(benchmarks, config);
var gcCollectionColumns = gcDiagnoser.GetColumns.OfType <GCDiagnoser.GCCollectionColumn>().ToArray();
var listStructEnumeratorBenchmark = benchmarks.Single(benchmark => benchmark.ShortInfo.Contains("ListStructEnumerator"));
var listObjectEnumeratorBenchmark = benchmarks.Single(benchmark => benchmark.ShortInfo.Contains("ListObjectEnumerator"));
const int gen0Index = 0;
var structEnumeratorGen0Collections = gcCollectionColumns[gen0Index].GetValue(summary, listStructEnumeratorBenchmark);
var objectEnumeratorGen0Collections = gcCollectionColumns[gen0Index].GetValue(summary, listObjectEnumeratorBenchmark);
Assert.Equal("-", structEnumeratorGen0Collections);
Assert.True(double.Parse(objectEnumeratorGen0Collections) > 0);
}
