internal ImmutableConfig(
ImmutableArray <IColumnProvider> uniqueColumnProviders,
ImmutableHashSet <ILogger> uniqueLoggers,
ImmutableHashSet <HardwareCounter> uniqueHardwareCounters,
ImmutableHashSet <IDiagnoser> uniqueDiagnosers,
ImmutableArray <IExporter> uniqueExporters,
ImmutableHashSet <IAnalyser> uniqueAnalyzers,
ImmutableHashSet <IValidator> uniqueValidators,
ImmutableHashSet <IFilter> uniqueFilters,
ImmutableHashSet <BenchmarkLogicalGroupRule> uniqueRules,
ImmutableHashSet <Job> uniqueRunnableJobs,
ConfigUnionRule unionRule,
string artifactsPath,
CultureInfo cultureInfo,
IOrderer orderer,
SummaryStyle summaryStyle,
ConfigOptions options)
{
columnProviders = uniqueColumnProviders;
loggers = uniqueLoggers;
hardwareCounters = uniqueHardwareCounters;
diagnosers = uniqueDiagnosers;
exporters = uniqueExporters;
analysers = uniqueAnalyzers;
validators = uniqueValidators;
filters = uniqueFilters;
rules = uniqueRules;
jobs = uniqueRunnableJobs;
UnionRule = unionRule;
ArtifactsPath = artifactsPath;
CultureInfo = cultureInfo;
Orderer = orderer;
SummaryStyle = summaryStyle;
Options = options;
}
public FullProfileConfig()
{
var summaryStyle = new SummaryStyle(false, SizeUnit.KB, TimeUnit.Millisecond, true);
this.SummaryStyle = summaryStyle;
Add(
DisassemblyDiagnoser.Create(
new DisassemblyDiagnoserConfig(printAsm: true, printPrologAndEpilog: true, recursiveDepth: 3, printDiff: true)
)
);
Add(MemoryDiagnoser.Default);
Add(
HardwareCounter.BranchInstructions,
//HardwareCounter.CacheMisses,
HardwareCounter.BranchMispredictions
);
Add(
Job
.Core
.With(Runtime.Core)
.WithWarmupCount(2)
.WithIterationCount(10)
.WithIterationTime(TimeInterval.FromMilliseconds(200))
.WithLaunchCount(1)
.With(new GcMode()
{
Force = false
})
);
}
/// <summary>
/// Gets column title formatted using the specified style
/// </summary>
public static string GetColumnTitle(this IColumn column, SummaryStyle style)
{
if (!style.PrintUnitsInHeader)
{
return(column.ColumnName);
}
switch (column.UnitType)
{
case UnitType.CodeSize:
return($"{column.ColumnName} [{style.CodeSizeUnit.Name}]");
case UnitType.Size:
return($"{column.ColumnName} [{style.SizeUnit.Name}]");
case UnitType.Time:
return($"{column.ColumnName} [{style.TimeUnit.Name}]");
case UnitType.Dimensionless:
return(column.ColumnName);
default:
return(column.ColumnName);
}
}
public Config()
{
var summaryStyle = new SummaryStyle(false, SizeUnit.KB, TimeUnit.Millisecond, true);
this.SummaryStyle = summaryStyle;
Add(MemoryDiagnoser.Default);
//Add(
// HardwareCounter.BranchInstructions,
// //HardwareCounter.CacheMisses,
// HardwareCounter.BranchMispredictions
//);
Add(
Job
.Core
.With(Runtime.Core)
.WithWarmupCount(2)
.WithIterationCount(30)
.WithIterationTime(TimeInterval.FromMilliseconds(200))
.WithLaunchCount(1)
.With(new GcMode()
{
Force = false
})
);
}
public string GetValue(Summary summary, BenchmarkCase benchmark, SummaryStyle style)
{
if (!summary.HasReport(benchmark))
{
return("-");
}
var results = summary[benchmark].ExecuteResults;
if (results.Count != 1)
{
return("-");
}
var result = results.Single();
var buffer = new StringBuilder();
foreach (var line in result.ExtraOutput)
{
if (Metric.TryParse(line, out Metric metric))
{
if (buffer.Length > 0)
{
buffer.Append(", ");
}
buffer.Append(metric.ToColumnValue());
}
}
return(buffer.Length > 0 ? buffer.ToString() : "-");
}
public List <double> GetAllValues(Summary summary, SummaryStyle style) => summary.Reports .Where(r => r.ResultStatistics != null) .Select(r => calc(r.ResultStatistics)) .Where(v => !double.IsNaN(v) && !double.IsInfinity(v)) .Select(v => UnitType == UnitType.Time ? v / style.TimeUnit.NanosecondAmount : v) .ToList();
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
var method = benchmarkCase.Descriptor.WorkloadMethod;
var instance = Activator.CreateInstance(method.DeclaringType);
var argument = benchmarkCase.Parameters.GetArgument("N").Value;
var result = method.Invoke(instance, new [] { argument });
if (method.Name.Contains("Serialize"))
{
switch (result)
{
case byte[] b:
return(ConvertToDisplay(b.LongLength));
case string s:
return(ConvertToDisplay(System.Text.Encoding.UTF8.GetBytes(s).LongLength));
default:
return("-");
}
}
else
{
switch (result)
{
case LabModel[] res:
return($"{res?.Length ?? 0}N");
default:
return("-");
}
}
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
var valueOfN = (int)benchmarkCase.Parameters.Items.Single(p => p.Name == "N").Value;
var timePerN = summary[benchmarkCase].ResultStatistics.Mean / valueOfN;
return(timePerN.ToTimeStr(benchmarkCase.Config.Encoding, TimeUnit.GetBestTimeUnit(timePerN)));
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
var benchmarkName = benchmarkCase.Descriptor.WorkloadMethod.Name.ToLower();
var filename = $"disk-size.{benchmarkName}.txt";
var path = $"C:\\test\\{filename}";
return(File.Exists(path) ? File.ReadAllText(path) : $"file not found");
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
var disp_info = benchmarkCase.DisplayInfo;
var nbFloat = int.Parse(benchmarkCase.Parameters.Items.FirstOrDefault(x => x.Name == "nbFloat").ToString());
var s = summary.Reports.Where(x => x.BenchmarkCase.DisplayInfo == disp_info).FirstOrDefault();
double fps = nbFloat * 1000 / s.ResultStatistics.Min;
return(string.Format("{0,8:f2}", fps));
}
private static string Format(Statistics statistics, SummaryStyle style)
{
if (statistics == null)
{
return("NA");
}
var d = 1000.0 / statistics.Mean;
return(double.IsNaN(d) ? "NA" : d.ToString("N2", style.CultureInfo));
}
public string GetValue(Summary summary, BenchmarkCase benchmark, SummaryStyle style)
{
var method = benchmark.Descriptor.WorkloadMethod;
var instance = Activator.CreateInstance(method.DeclaringType);
var hash = ((byte[])method.Invoke(instance, new[] { benchmark.Parameters[0].Value })).ToHexString();
if (hash.Length > 16)
{
hash = hash.Substring(0, 12) + "...";
}
return(hash);
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
var disp_info = benchmarkCase.DisplayInfo;
var fileName = benchmarkCase.Parameters.Items.FirstOrDefault(x => x.Name == "FileName").ToString();
var volume = new FileInfo(fileName).Length / 1024;
var s = summary.Reports.Where(x => x.BenchmarkCase.DisplayInfo == disp_info).FirstOrDefault();
double vps = volume * 1000000 / s.ResultStatistics.Min;
return(string.Format("{0,8:f2}", vps));
}
public override void ExportToLog(Summary summary, ILogger logger)
{
// We construct HostEnvironmentInfo manually, so that we can have the HardwareTimerKind enum as text, rather than an integer
// SimpleJson serialiser doesn't seem to have an enum String/Value option (to-be-fair, it is meant to be "Simple")
var environmentInfo = new
{
HostEnvironmentInfo.BenchmarkDotNetCaption,
summary.HostEnvironmentInfo.BenchmarkDotNetVersion,
OsVersion = summary.HostEnvironmentInfo.OsVersion.Value,
summary.HostEnvironmentInfo.CpuInfo,
summary.HostEnvironmentInfo.RuntimeVersion,
summary.HostEnvironmentInfo.Architecture,
summary.HostEnvironmentInfo.HasAttachedDebugger,
summary.HostEnvironmentInfo.HasRyuJit,
summary.HostEnvironmentInfo.Configuration,
summary.HostEnvironmentInfo.JitInfo,
DotNetCliVersion = summary.HostEnvironmentInfo.DotNetSdkVersion.Value,
summary.HostEnvironmentInfo.ChronometerFrequency,
HardwareTimerKind = summary.HostEnvironmentInfo.HardwareTimerKind.ToString()
};
// If we just ask SimpleJson to serialise the entire "summary" object it throws several errors.
// So we are more specific in what we serialise (plus some fields/properties aren't relevant)
var columns = summary.GetColumns()
// exclude
.Where(col => !(col is BenchmarkDotNet.Columns.StatisticColumn || col is BenchmarkDotNet.Columns.TargetMethodColumn || col is BenchmarkDotNet.Columns.ParamColumn))
.ToArray();
var summaryStyle = new SummaryStyle(CultureInfo.InvariantCulture, printUnitsInHeader: true, printUnitsInContent: false, sizeUnit: SizeUnit.B, timeUnit: TimeUnit.Nanosecond);
var benchmarks = summary.Reports.Select(r =>
{
var data = new Dictionary <string, object> {
// We don't need Benchmark.ShortInfo, that info is available via Benchmark.Parameters below
{ "DisplayInfo", r.BenchmarkCase.DisplayInfo },
{ "Namespace", r.BenchmarkCase.Descriptor.Type.Namespace },
{ "Type", r.BenchmarkCase.Descriptor.Type.Name },
{ "Method", r.BenchmarkCase.Descriptor.WorkloadMethod.Name },
{ "MethodTitle", r.BenchmarkCase.Descriptor.WorkloadMethodDisplayInfo },
{ "Parameters", r.BenchmarkCase.Parameters.Items.ToDictionary(p => p.Name, p => p.Value) },
// { "Properties", r.Benchmark.Job.ToSet().ToDictionary(p => p.Name, p => p.Value) }, // TODO
{ "Statistics", r.ResultStatistics },
{ "Columns", columns.Where(col => !col.IsDefault(summary, r.BenchmarkCase)).ToDictionary(col => col.Id, col => col.GetValue(summary, r.BenchmarkCase, summaryStyle)) }
};
// We show MemoryDiagnoser's results only if it is being used
if (config.GetDiagnosers().OfType <SynchronousMemoryDiagnoser>().FirstOrDefault() is SynchronousMemoryDiagnoser smd &&
smd.FindGCStats(r.BenchmarkCase) is GcStats gcStats)
{
data.Add("Memory", gcStats);
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
var benchmarkName = benchmarkCase.Descriptor.WorkloadMethod.Name.ToLower();
var parameter = benchmarkCase.Parameters.Items.FirstOrDefault(x => x.Name == "N");
if (parameter == null)
{
return("no parameter");
}
var N = Convert.ToInt32(parameter.Value);
var filename = Path.Combine(DbSizeOutputFolder, $"disk-size.{benchmarkName}.{N}.txt");
return(File.Exists(filename) ? File.ReadAllText(filename) : "no file");
}
public void UserCanDefineCustomSummaryStyle()
{
var summaryStyle = new SummaryStyle
(
printUnitsInHeader: true,
printUnitsInContent: false,
sizeUnit: SizeUnit.B,
timeUnit: TimeUnit.Millisecond
);
var config = ManualConfig.CreateEmpty().With(summaryStyle);
Assert.True(config.SummaryStyle.PrintUnitsInHeader);
Assert.False(config.SummaryStyle.PrintUnitsInContent);
Assert.Equal(SizeUnit.B, config.SummaryStyle.SizeUnit);
Assert.Equal(TimeUnit.Millisecond, config.SummaryStyle.TimeUnit);
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
System.Reflection.MethodInfo mi = benchmarkCase.Descriptor.WorkloadMethod;
if (mi.Name.Contains("Serialize"))
{
var instance = Activator.CreateInstance(mi.DeclaringType);
mi.DeclaringType.GetField("Serializer").SetValue(instance, benchmarkCase.Parameters[0].Value);
mi.DeclaringType.GetMethod("Setup").Invoke(instance, null);
var bytes = (byte[])mi.Invoke(instance, null);
return(ToHumanReadableSize(bytes.Length));
}
else
{
return("-");
}
}
public void UserCanDefineCusomSummaryStyle()
{
var summaryStyle = new SummaryStyle
{
PrintUnitsInHeader = true,
PrintUnitsInContent = false,
SizeUnit = SizeUnit.B,
TimeUnit = TimeUnit.Millisecond
};
var config = ManualConfig.CreateEmpty().With(summaryStyle);
Assert.True(config.GetSummaryStyle().PrintUnitsInHeader);
Assert.False(config.GetSummaryStyle().PrintUnitsInContent);
Assert.Equal(SizeUnit.B, config.GetSummaryStyle().SizeUnit);
Assert.Equal(TimeUnit.Millisecond, config.GetSummaryStyle().TimeUnit);
}
private static Summary CreateMockSummary(bool printUnitsInContent, bool printUnitsInHeader, TimeUnit timeUnit, double metricValue)
{
var summaryStyle = new SummaryStyle(TestCultureInfo.Instance, printUnitsInHeader, null, timeUnit, printUnitsInContent);
var config = new ManualConfig().WithSummaryStyle(summaryStyle);
var benchmarkCase = new BenchmarkCase(
new Descriptor(null, null),
Job.Dry,
new ParameterInstances(ImmutableArray <ParameterInstance> .Empty),
ImmutableConfigBuilder.Create(config));
var metric = new Metric(LocalMetricDescriptor.TimeInstance, metricValue);
var benchmarkReport = new BenchmarkReport(true, benchmarkCase, null, null, null, new List <Metric>()
{
metric
});
return(new Summary("", new[] { benchmarkReport }.ToImmutableArray(), HostEnvironmentInfo.GetCurrent(),
"", "", TimeSpan.Zero, CultureInfo.InvariantCulture, ImmutableArray <ValidationError> .Empty, ImmutableArray <IColumnHidingRule> .Empty));
}
public void Add(IConfig config)
{
columnProviders.AddRange(config.GetColumnProviders());
exporters.AddRange(config.GetExporters());
loggers.AddRange(config.GetLoggers());
diagnosers.AddRange(config.GetDiagnosers());
analysers.AddRange(config.GetAnalysers());
jobs.AddRange(config.GetJobs());
validators.AddRange(config.GetValidators());
hardwareCounters.AddRange(config.GetHardwareCounters());
filters.AddRange(config.GetFilters());
Orderer = config.Orderer ?? Orderer;
ArtifactsPath = config.ArtifactsPath ?? ArtifactsPath;
CultureInfo = config.CultureInfo ?? CultureInfo;
SummaryStyle = config.SummaryStyle ?? SummaryStyle;
logicalGroupRules.AddRange(config.GetLogicalGroupRules());
Options |= config.Options;
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
var disp_info = benchmarkCase.DisplayInfo;
var fileName = benchmarkCase.Parameters.Items.FirstOrDefault(x => x.Name == "FileName").ToString();
var volume = 0;
foreach (string l in System.IO.File.ReadAllLines(fileName))
{
volume += l.Length;
}
var s = summary.Reports.Where(x => x.BenchmarkCase.DisplayInfo == disp_info).FirstOrDefault();
double vps = volume * 1000 / s.ResultStatistics.Min;
return(string.Format("{0,8:f2}", vps));
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
var methodName = benchmarkCase.Descriptor.WorkloadMethod.GetCustomAttribute <PayloadSizeColumnAttribute>()?.MethodName;
if (string.IsNullOrEmpty(methodName))
{
return("-");
}
var benchmark = Activator.CreateInstance(benchmarkCase.Descriptor.Type);
var method = benchmark
.GetType()
.GetMethod(methodName, BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.Instance)
.CreateDelegate <Func <ValueTask <long> > >(benchmark);
var size = method().Result;
return(LengthToString(size));
}
public Config()
{
var summaryStyle = new SummaryStyle(false, SizeUnit.KB, TimeUnit.Millisecond, true);
this.SummaryStyle = summaryStyle;
Add(MemoryDiagnoser.Default);
Add(
Job
.Default
.WithWarmupCount(2)
.WithIterationCount(25)
.WithIterationTime(TimeInterval.FromMilliseconds(200))
.With(new GcMode()
{
Force = false
})
.WithLaunchCount(1)
);
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
var type = benchmarkCase.Descriptor.Type;
var report = summary[benchmarkCase];
if (!type.IsSubclassOf(typeof(APredefinedBenchmark)))
{
return(string.Empty);
}
if (!report.Success || !report.BuildResult.IsBuildSuccess || report.ExecuteResults.Count == 0)
{
return("NA");
}
var instance = (APredefinedBenchmark)Activator.CreateInstance(type);
if (instance == null)
{
return("NA");
}
var paramInstances = benchmarkCase.Parameters;
instance.MessageTarget = UpdateInstanceValueSave(instance.MessageTarget, paramInstances, nameof(instance.MessageTarget));
int messageCount = instance.MessageTarget;
var statistics = report.ResultStatistics;
var meanSeconds = TimeUnit.Convert(statistics.Mean, TimeUnit.Nanosecond, TimeUnit.Second);
var msgPerSecond = messageCount / meanSeconds;
var cultureInfo = summary.GetCultureInfo();
if (style.PrintUnitsInContent)
{
return(msgPerSecond.ToString("N0", cultureInfo) + " msg/s");
}
return(msgPerSecond.ToString("N0", cultureInfo));
}
public Config()
{
var summaryStyle = new SummaryStyle(false, SizeUnit.KB, TimeUnit.Millisecond, true);
this.SummaryStyle = summaryStyle;
Add(
HardwareCounter.BranchInstructions,
//HardwareCounter.CacheMisses,
HardwareCounter.BranchMispredictions
);
Add(
Job
.Default
.With(Runtime.Core)
.WithWarmupCount(5)
.WithIterationCount(10)
.WithIterationTime(TimeInterval.FromMilliseconds(100))
.WithLaunchCount(1)
);
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style)
{
System.Reflection.MethodInfo mi = benchmarkCase.Descriptor.WorkloadMethod;
if (!mi.Name.Contains("Serialize"))
{
return("-");
}
var instance = Activator.CreateInstance(mi.DeclaringType);
mi.DeclaringType.GetField("Serializer").SetValue(instance, benchmarkCase.Parameters[0].Value);
mi.DeclaringType.GetMethod("Setup").Invoke(instance, null);
var bytes = (byte[])mi.Invoke(instance, null);
var byteSize = bytes.Length;
var cultureInfo = summary.GetCultureInfo();
if (style.PrintUnitsInContent)
{
return(SizeValue.FromBytes(byteSize).ToString(style.SizeUnit, cultureInfo));
}
return(byteSize.ToString("0.##", cultureInfo));
}
public BenchmarkConfig()
{
CultureInfo cultureInfo = (CultureInfo)CultureInfo.InvariantCulture.Clone();
cultureInfo.NumberFormat.NumberDecimalSeparator = ".";
cultureInfo.NumberFormat.NumberGroupSeparator = "";
SummaryStyle summaryStyle = new SummaryStyle(
cultureInfo: cultureInfo,
printUnitsInHeader: true,
sizeUnit: BenchmarkDotNet.Columns.SizeUnit.B,
timeUnit: Perfolizer.Horology.TimeUnit.Nanosecond,
printUnitsInContent: false
);
CultureInfo = cultureInfo;
AddExporter(new CsvExporter(CsvSeparator.Comma, summaryStyle));
AddExporter(new CsvMeasurementsExporter(CsvSeparator.Comma, summaryStyle));
Job job = Job.Default
.WithIterationCount(50);
//.WithAffinity((IntPtr)0b0000_0001); // be sure to stay on core #1
if (System.Diagnostics.Debugger.IsAttached)
{
job = job.WithToolchain(new InProcessEmitToolchain(
TimeSpan.FromHours(1), // 1h should be enough to debug the benchmark
true));
Options |= ConfigOptions.DisableOptimizationsValidator;
}
AddJob(job);
}
public static CsvMeasurementsExporter WithStyle(SummaryStyle style) => new CsvMeasurementsExporter(CsvSeparator.CurrentCulture, style);
private static IEnumerable <ParameterInstances> GetArgumentsDefinitions(MethodInfo benchmark, Type target, SummaryStyle summaryStyle)
{
var argumentsAttributes = benchmark.GetCustomAttributes <PriorityAttribute>();
int priority = argumentsAttributes.Select(attribute => attribute.Priority).Sum();
var parameterDefinitions = benchmark.GetParameters()
.Select(parameter => new ParameterDefinition(parameter.Name, false, Array.Empty <object>(), true, parameter.ParameterType, priority))
.ToArray();
if (parameterDefinitions.IsEmpty())
{
yield return(new ParameterInstances(Array.Empty <ParameterInstance>()));
yield break;
}
foreach (var argumentsAttribute in benchmark.GetCustomAttributes <ArgumentsAttribute>())
{
if (parameterDefinitions.Length != argumentsAttribute.Values.Length)
{
throw new InvalidOperationException($"Benchmark {benchmark.Name} has invalid number of defined arguments provided with [Arguments]! {argumentsAttribute.Values.Length} instead of {parameterDefinitions.Length}.");
}
yield return(new ParameterInstances(
argumentsAttribute
.Values
.Select((value, index) =>
{
var definition = parameterDefinitions[index];
var type = definition.ParameterType;
return new ParameterInstance(definition, Map(value, type), summaryStyle);
})
.ToArray()));
}
if (!benchmark.HasAttribute <ArgumentsSourceAttribute>())
{
yield break;
}
var argumentsSourceAttribute = benchmark.GetCustomAttribute <ArgumentsSourceAttribute>();
var valuesInfo = GetValidValuesForParamsSource(target, argumentsSourceAttribute.Name);
for (int sourceIndex = 0; sourceIndex < valuesInfo.values.Length; sourceIndex++)
{
yield return(SmartParamBuilder.CreateForArguments(benchmark, parameterDefinitions, valuesInfo, sourceIndex, summaryStyle));
}
}
public string GetValue(Summary summary, BenchmarkCase benchmarkCase, SummaryStyle style) => GetValue(summary, benchmarkCase);
