private BenchmarkCase(Descriptor descriptor, Job job, ParameterInstances parameters, ImmutableConfig config)
{
Descriptor = descriptor;
Job = job;
Parameters = parameters;
Config = config;
}
private static string GetBenchmarkParameters(MethodInfo method, ParameterInstances benchmarkParameters)
{
var methodParameters = method.GetParameters();
var parametersBuilder = new StringBuilder(methodParameters.Length * 20).Append('(');
for (int i = 0; i < methodParameters.Length; i++)
{
if (i > 0)
{
parametersBuilder.Append(", ");
}
parametersBuilder.Append(methodParameters[i].Name).Append(':').Append(' ');
parametersBuilder.Append(GetArgument(benchmarkParameters.GetArgument(methodParameters[i].Name).Value, methodParameters[i].ParameterType));
}
foreach (var parameter in benchmarkParameters.Items.Where(parameter => !parameter.IsArgument))
{
if (methodParameters.Length > 0)
{
parametersBuilder.Append(", ");
}
parametersBuilder.Append(parameter.Name).Append(':').Append(' ');
parametersBuilder.Append(GetArgument(parameter.Value, parameter.Value?.GetType()));
}
return(parametersBuilder.Append(')').ToString());
}
public ValidationResult(ParameterInstances testCase, IBenchmarkValidator validator, string message, bool isViolation)
{
this.TestCase = testCase;
this.Validator = validator;
this.Message = message;
this.IsViolation = isViolation;
}
private static string GetSessionName(string prefix, Benchmark benchmark, ParameterInstances parameters = null)
{
if (parameters != null && parameters.Items.Count > 0)
{
return($"{prefix}-{benchmark.FolderInfo}-{parameters.FolderInfo}");
}
return($"{prefix}-{benchmark.FolderInfo}");
}
protected string GetSessionName(string prefix, Benchmark benchmark, ParameterInstances parameters = null)
{
if (parameters != null && parameters.Items.Count > 0)
{
return($"{prefix}-{benchmark.ShortInfo}-{parameters.FullInfo}");
}
return($"{prefix}-{benchmark.ShortInfo}");
}
private static T UpdateInstanceValueSave <T>(T current, ParameterInstances instances, string name)
{
var instance = instances.Items.FirstOrDefault(item => item.Name == name);
if (instance == null)
{
// Parameter not present, keep the current value
return(current);
}
return((T)instance.Value);
}
public ExternalProcessBenchmark(
string workingDir,
string arguments,
Func <string, int> buildFunc,
Job job,
ParameterInstances parameterInstances)
: base(GetTarget(), job, parameterInstances)
{
WorkingDirectory = workingDir;
Arguments = arguments;
BuildFunc = buildFunc;
}
public static bool Match(this ParameterInstances parameters, IDictionary <string, IEnumerable <string> > patterns)
{
foreach (var pattern in patterns)
{
foreach (var parameter in parameters.Items)
{
if (parameter.Name.Equals(pattern.Key, StringComparison.OrdinalIgnoreCase) &&
!Match(parameter.Value.ToString(), pattern.Value))
{
return(false);
}
}
}
return(true);
}
static void Main(string[] args)
{
// Initialize common settings
var job = Job.ShortRun
.With(InProcessEmitToolchain.Instance) // The current .NET Core toolchain requires the DLLs to be named exactly as their projects
.WithBaseline(true);
var parameters = new ParameterInstances(Array.Empty <ParameterInstance>());
var config = DefaultConfig.Instance
.With(ConfigOptions.DisableOptimizationsValidator)
.With(MemoryDiagnoser.Default)
.With(TargetMethodColumn.Type)
.With(ConfigurationColumn.Instance)
.With(BenchmarkLogicalGroupRule.ByCategory)
.CreateImmutableConfig();
var o1Assembly = typeof(TransformationsO1.Helper).Assembly;
var releaseAssembly = typeof(TransformationsRelease.Helper).Assembly;
// Add a method from each class in both O1 (optimized, but without transformations) and Release version
var cases = new List <BenchmarkCase>();
foreach (var o1Type in o1Assembly.GetTypes())
{
var o1Method = o1Type.GetMethod("run");
if (o1Method == null)
{
continue;
}
var releaseType = releaseAssembly.GetType(o1Type.FullName);
var releaseMethod = releaseType.GetMethod("run");
cases.Add(BenchmarkCase.Create(new Descriptor(o1Type, o1Method), job, parameters, config));
cases.Add(BenchmarkCase.Create(new Descriptor(releaseType, releaseMethod), job, parameters, config));
}
var runInfo = new BenchmarkRunInfo(cases.ToArray(), typeof(TransformationsO1.Helper), config);
BenchmarkRunner.Run(runInfo);
}
public Benchmark(Target target, IJob job, ParameterInstances parameters)
{
Target = target;
Job = job;
Parameters = parameters;
}
public static BenchmarkCase Create(Descriptor descriptor, Job job, ParameterInstances parameters, ImmutableConfig config) => new BenchmarkCase(descriptor, job.MakeSettingsUserFriendly(descriptor), parameters, config);
public static Benchmark Create(Target target, Job job, ParameterInstances parameters) => new Benchmark(target, job.MakeSettingsUserFriendly(target), parameters);
private Benchmark(Target target, Job job, ParameterInstances parameters)
{
Target = target;
Job = job;
Parameters = parameters;
}
private BenchmarkCase(Descriptor descriptor, Job job, ParameterInstances parameters)
{
Descriptor = descriptor;
Job = job;
Parameters = parameters;
}
private ValidationResult GetValidationResult(ParameterInstances parameterInstances, BenchmarkResults.BeforeAndAfter resultMeasurement)
{
double hypothesizedDifference;
if (this.byAtLeastTimeInterval != null)
{
hypothesizedDifference = byAtLeastTimeInterval.Value.Nanoseconds;
}
else if (this.byAtLeastPercent != null)
{
var baselineMean = resultMeasurement.Baseline.ResultStatistics.Mean;
hypothesizedDifference = baselineMean * this.byAtLeastPercent.Value.Multiplier;
}
else
{
throw new InvalidOperationException("This is why you use a library like OneOf");
}
switch (alternateHypothesis)
{
case TwoSampleHypothesis.FirstValueIsGreaterThanSecond:
// observed: baseline - treatment -- we are saying First<Second so baseline - treatment should be negative
hypothesizedDifference *= -1;
break;
case TwoSampleHypothesis.FirstValueIsSmallerThanSecond:
break;
default:
throw new ArgumentOutOfRangeException();
}
var testResult = new TwoSampleAutoTOrZTestHypothesisTest()
.TestHypothesis(
resultMeasurement,
hypothesizedDifference,
alternateHypothesis,
this.alpha);
var isMatch = testResult.IsSignificant;
var confidenceInterval = testResult.ConfidenceInterval;
var observedDifference = testResult.ObservedDifference;
var confIntervalInMs = new DoubleRange(confidenceInterval.Min * 1e-6, confidenceInterval.Max * 1e-6);
var confidenceLevel = 1 - this.alpha;
string byAtLeastString;
if (byAtLeastPercent != null)
{
byAtLeastString = this.byAtLeastPercent.Value.Multiplier.ToString("P0");
}
else if (byAtLeastTimeInterval != null)
{
byAtLeastString = this.byAtLeastTimeInterval.Value.ToString();
}
else
{
throw new Exception();
}
var message =
$"Support: {(isMatch ? "do support" : "cannot support")}\r\n" +
$"{this.alternateHypothesis.ToDescriptiveString("baseline duration", "treatment duration")} by {byAtLeastString}\r\n" +
$"Alpha: {this.alpha}.\r\n" +
$"HypothesizedDifference: {hypothesizedDifference}.\r\n" +
$"ObservedDifference: {observedDifference}\r\n" +
$"ConfidenceInterval: {confIntervalInMs} ms\r\n" +
$"Baseline {resultMeasurement.Baseline.ResultStatistics.ToSummaryString(confidenceLevel)}" +
$"Treatment {resultMeasurement.Treatment.ResultStatistics.ToSummaryString(confidenceLevel)}";
return(new ValidationResult(
parameterInstances,
this,
message,
// TODO: P3 - We are abusing this type here... isViolation != isMatch
isViolation: isMatch));
}
