/// <summary>
/// Creates job for the competition. <see cref="BenchmarkDotNet.Characteristics.CharacteristicObject.Frozen"/> is false.
/// </summary>
/// <param name="jobId">The job identifier.</param>
/// <param name="metadataSource">The metadata source.</param>
/// <param name="competitionFeatures">The competition features.</param>
/// <returns>
/// New job for the competition. <see cref="BenchmarkDotNet.Characteristics.CharacteristicObject.Frozen"/> is false.
/// </returns>
protected virtual Job CreateJobUnfrozen(
[CanBeNull] string jobId,
[CanBeNull] ICustomAttributeProvider metadataSource,
[NotNull] CompetitionFeatures competitionFeatures)
{
var platform = competitionFeatures.ResolveValueAsNullable(CompetitionFeatures.PlatformCharacteristic);
if (jobId != null)
{
jobId += platform;
}
var job = new Job(jobId, DefaultJob);
if (platform != null)
{
job.Env.Platform = platform.GetValueOrDefault();
}
if (competitionFeatures.BurstMode)
{
job.Apply(BurstModeModifier);
job.Run.TargetCount *= 2;
}
if (competitionFeatures.TroubleshootingMode)
{
job.Apply(new InfrastructureMode {
Toolchain = InProcessToolchain.Instance
});
}
return(job);
}
protected ICompetitionConfig CreateCore(
[CanBeNull] ICustomAttributeProvider metadataSource,
[CanBeNull] CompetitionFeatures competitionFeatures)
{
// 1. Check for existing config
var config = metadataSource?.TryGetMetadataAttribute <ICompetitionConfigSource>()?.Config;
if (config != null)
{
return(CompleteConfig(new ManualCompetitionConfig(config)));
}
// 2. Get competition features
// TODO: redesign? Do not lost features Id
competitionFeatures = competitionFeatures?.UnfreezeCopy()
?? CreateCompetitionFeaturesUnfrozen(ConfigId, metadataSource);
competitionFeatures = CompleteFeatures(competitionFeatures);
// 3. Create config stub
var result = CreateEmptyConfig(metadataSource, competitionFeatures);
// 4. Create job
result.Add(CreateJobUnfrozen(ConfigId, metadataSource, competitionFeatures));
// 5. Create competition options
result.Set(CreateCompetitionOptionsUnfrozen(metadataSource, competitionFeatures));
// 6. Apply config sources.
ApplyConfigSources(result, metadataSource);
// 7. Apply competition modifiers
ApplyCompetitionModifiers(result, metadataSource);
return(CompleteConfig(result));
}
protected override Job CreateJobUnfrozen(
string jobId, ICustomAttributeProvider metadataSource,
CompetitionFeatures competitionFeatures)
{
var result = base.CreateJobUnfrozen(jobId, metadataSource, competitionFeatures);
result.Apply(
new Job()
{
Run =
{
LaunchCount = 1,
WarmupCount = 2,
TargetCount = 2,
InvocationCount = 1,
UnrollFactor = 1
},
Env =
{
Affinity = new IntPtr(-1)
}
});
return(result);
}
protected override ManualCompetitionConfig CreateEmptyConfig(
ICustomAttributeProvider metadataSource, CompetitionFeatures competitionFeatures)
{
var result = base.CreateEmptyConfig(metadataSource, competitionFeatures);
result.Metrics.RemoveAll(m => !m.IsPrimaryMetric);
return(result);
}
public static ICompetitionConfig FindFactoryAndCreate(
[CanBeNull] Type benchmarkType,
[CanBeNull] CompetitionFeatures competitionFeatures)
{
var factory = benchmarkType?.TryGetMetadataAttribute <ICompetitionConfigFactorySource>()?.ConfigFactory
?? FallbackInstance;
return(factory.Create(benchmarkType, competitionFeatures));
}
public static ICompetitionConfig FindFactoryAndCreate(
[CanBeNull] Assembly targetAssembly,
[CanBeNull] CompetitionFeatures competitionFeatures)
{
var factory = targetAssembly?.TryGetMetadataAttribute <ICompetitionConfigFactorySource>()?.ConfigFactory
?? FallbackInstance;
return(factory.Create(targetAssembly, competitionFeatures));
}
protected override CompetitionOptions CreateCompetitionOptionsUnfrozen(
ICustomAttributeProvider metadataSource, CompetitionFeatures competitionFeatures)
{
var result = base.CreateCompetitionOptionsUnfrozen(metadataSource, competitionFeatures);
result.RunOptions.AllowDebugBuilds = true;
return(result);
}
public IReadOnlyDictionary <Type, CompetitionState> Run(
[NotNull] Type[] benchmarkTypes,
[CanBeNull] CompetitionFeatures competitionFeatures = null)
{
var result = new Dictionary <Type, CompetitionState>();
foreach (var benchmarkType in benchmarkTypes)
{
result[benchmarkType] = RunCore(benchmarkType, null, competitionFeatures);
}
return(result);
}
/// <summary>Creates job for the competition. <see cref="Job.Frozen"/> is false.</summary>
/// <param name="jobId">The job identifier.</param>
/// <param name="metadataSource">The metadata source.</param>
/// <param name="competitionFeatures">The competition features.</param>
/// <returns>New job for the competition. <see cref="Job.Frozen"/> is false.</returns>
protected override Job CreateJobUnfrozen(
string jobId,
ICustomAttributeProvider metadataSource,
CompetitionFeatures competitionFeatures)
{
var result = base.CreateJobUnfrozen(jobId, metadataSource, competitionFeatures);
if (competitionFeatures.Platform == Platform.X64)
{
result.Apply(EnvMode.RyuJitX64);
}
return(result);
}
private CompetitionState RunCore(
[NotNull] Type benchmarkType,
[CanBeNull] ICompetitionConfig competitionConfig,
[CanBeNull] CompetitionFeatures competitionFeatures)
{
Code.NotNull(benchmarkType, nameof(benchmarkType));
competitionConfig = CreateBenchmarkConfig(benchmarkType, competitionConfig, competitionFeatures);
var hostLogger = competitionConfig.GetLoggers().OfType <HostLogger>().Single();
var previousDirectory = Environment.CurrentDirectory;
var currentDirectory = GetOutputDirectory(benchmarkType.Assembly);
if (currentDirectory == previousDirectory)
{
currentDirectory = null;
previousDirectory = null;
}
CompetitionState competitionState = null;
try
{
SetCurrentDirectoryIfNotNull(currentDirectory);
try
{
competitionState = CompetitionCore.Run(benchmarkType, competitionConfig);
ProcessRunComplete(competitionState);
}
finally
{
ReportHostLogger(hostLogger, competitionState?.LastRunSummary);
}
using (FilteringLogger.BeginLogImportant(competitionState.Config))
{
ReportMessagesToUser(competitionState);
}
}
finally
{
LoggerHelpers.FlushLoggers(competitionConfig);
SetCurrentDirectoryIfNotNull(previousDirectory);
}
return(competitionState);
}
protected virtual ManualCompetitionConfig CreateEmptyConfig(
[CanBeNull] ICustomAttributeProvider metadataSource,
[NotNull] CompetitionFeatures competitionFeatures)
{
var result = new ManualCompetitionConfig(BenchmarkDotNet.Configs.DefaultConfig.Instance);
// DONTTOUCH: competition should not use default
// Jobs, Diagnosers, Exporters and Loggers.
// These omitted intentionally.
result.Jobs.Clear();
result.Diagnosers.Clear();
result.Exporters.Clear();
result.Loggers.Clear();
// TODO: better columns.
// TODO: columns for options.
result.ColumnProviders.Clear();
result.Add(competitionFeatures.TroubleshootingMode ? TroubleshootingModeColumns : DefaultColumns);
var targetAssembly = GetAssembly(metadataSource);
if (targetAssembly != null)
{
if (competitionFeatures.TroubleshootingMode)
{
result.Add(
GetImportantInfoLogger(targetAssembly),
GetDetailedLogger(targetAssembly));
result.Add(Exporters.CsvTimingsExporter.Default);
}
else
{
if (competitionFeatures.ImportantInfoLogger || competitionFeatures.ContinuousIntegrationMode)
{
result.Add(GetImportantInfoLogger(targetAssembly));
}
if (competitionFeatures.DetailedLogger)
{
result.Add(GetDetailedLogger(targetAssembly));
}
}
}
result.Add(DefaultMetrics.ToArray());
return(result);
}
private ICompetitionConfig CreateBenchmarkConfig(
[NotNull] Type benchmarkType,
ICompetitionConfig competitionConfig,
CompetitionFeatures competitionFeatures)
{
// ReSharper disable once UseObjectOrCollectionInitializer
var result = new ManualCompetitionConfig(
competitionConfig ??
CompetitionConfigFactory.FindFactoryAndCreate(benchmarkType, competitionFeatures));
InitCompetitionConfigOverride(result);
FixCompetitionConfig(result);
return(result.AsReadOnly());
}
// TODO: pass id?
/// <summary>
/// Creates options for the competition. The <see cref="BenchmarkDotNet.Characteristics.CharacteristicObject.Frozen"/> is false.
/// </summary>
/// <param name="metadataSource">The metadata source.</param>
/// <param name="competitionFeatures">The competition features.</param>
/// <returns>
/// Options for the competition. The <see cref="BenchmarkDotNet.Characteristics.CharacteristicObject.Frozen"/> is false.
/// </returns>
protected virtual CompetitionOptions CreateCompetitionOptionsUnfrozen(
[CanBeNull] ICustomAttributeProvider metadataSource,
[NotNull] CompetitionFeatures competitionFeatures)
{
var result = new CompetitionOptions(DefaultCompetitionOptions);
if (competitionFeatures.AnnotateSources)
{
result.Adjustments.AdjustMetrics = true;
if (competitionFeatures.IgnoreExistingAnnotations)
{
result.Annotations.IgnoreExistingAnnotations = true;
result.Annotations.PreviousRunLogUri = null;
}
else
{
result.Annotations.PreviousRunLogUri = competitionFeatures.PreviousRunLogUri;
}
}
if (Debugger.IsAttached)
{
result.RunOptions.AllowDebugBuilds = true;
}
if (competitionFeatures.TroubleshootingMode)
{
result.RunOptions.AllowDebugBuilds = true;
result.RunOptions.ReportWarningsAsErrors = false;
result.RunOptions.DetailedLogging = true;
}
else if (competitionFeatures.ReportWarningsAsErrors)
{
result.RunOptions.ReportWarningsAsErrors = true;
}
if (competitionFeatures.ContinuousIntegrationMode)
{
result.RunOptions.ContinuousIntegrationMode = true;
result.Annotations.LogAnnotations = true;
result.Annotations.PreviousRunLogUri = null;
result.Annotations.DontSaveUpdatedAnnotations = true;
}
return(result);
}
/// <summary>
/// Creates options for the competition. The <see cref="BenchmarkDotNet.Characteristics.CharacteristicObject.Frozen"/> is false.
/// </summary>
/// <param name="metadataSource">The metadata source.</param>
/// <param name="competitionFeatures">The competition features.</param>
/// <returns>
/// Options for the competition. The <see cref="BenchmarkDotNet.Characteristics.CharacteristicObject.Frozen"/> is false.
/// </returns>
protected override CompetitionOptions CreateCompetitionOptionsUnfrozen(
ICustomAttributeProvider metadataSource, CompetitionFeatures competitionFeatures)
{
var result = base.CreateCompetitionOptionsUnfrozen(metadataSource, competitionFeatures);
if (competitionFeatures.ContinuousIntegrationMode)
{
if (!result.HasValue(CompetitionAdjustmentMode.SkipRunsBeforeAdjustmentCharacteristic))
{
result.Adjustments.SkipRunsBeforeAdjustment = 1;
}
}
if (!result.HasValue(CompetitionAdjustmentMode.ForceEmptyMetricAdjustmentCharacteristic))
{
result.Adjustments.ForceEmptyMetricsAdjustment = true;
}
return(result);
}
/// <summary>Creates competition config for type.</summary>
/// <param name="benchmarkType">Benchmark class to run.</param>
/// <param name="competitionFeatures">The competition features.</param>
/// <returns>Competition config for type.</returns>
public ICompetitionConfig Create(
Type benchmarkType,
CompetitionFeatures competitionFeatures) =>
CreateCore(benchmarkType, competitionFeatures);
public IReadOnlyDictionary <Type, CompetitionState> Run(
[NotNull] Assembly assembly,
[CanBeNull] CompetitionFeatures competitionFeatures = null) =>
Run(BenchmarkHelpers.GetBenchmarkTypes(assembly), competitionFeatures);
/// <summary>Initializes a new instance of the <see cref="CompetitionFeaturesAttribute"/> class.</summary>
/// <param name="features">The features.</param>
protected CompetitionFeaturesAttribute([NotNull] CompetitionFeatures features)
{
Code.NotNull(features, nameof(features));
_features = new CompetitionFeatures(features);
}
/// <summary>Initializes a new instance of the <see cref="CompetitionFeaturesAttribute"/> class.</summary>
public CompetitionFeaturesAttribute()
{
_features = new CompetitionFeatures();
}
public static CompetitionState Run(
[NotNull] Type benchmarkType,
[CanBeNull] CompetitionFeatures competitionFeatures = null) =>
Runner.Run(benchmarkType, competitionFeatures);
/// <summary>Creates competition config for type.</summary>
/// <param name="targetAssembly">Assembly to create config for.</param>
/// <param name="competitionFeatures">The competition features.</param>
/// <returns>Competition config for type.</returns>
public ICompetitionConfig Create(
Assembly targetAssembly,
CompetitionFeatures competitionFeatures) =>
CreateCore(targetAssembly, competitionFeatures);
public CompetitionState Run(
[NotNull] Type benchmarkType,
[CanBeNull] CompetitionFeatures competitionFeatures = null) =>
RunCore(benchmarkType, null, competitionFeatures);
public CompetitionState Run <T>([CanBeNull] CompetitionFeatures competitionFeatures = null)
where T : class =>
RunCore(typeof(T), null, competitionFeatures);
public static ICompetitionConfig CreateConfig(
[NotNull] Type benchmarkType,
[CanBeNull] CompetitionFeatures competitionFeatures = null) =>
CompetitionConfigFactory.FindFactoryAndCreate(benchmarkType, competitionFeatures);
/// <summary>Completes competition features creation.</summary>
/// <param name="competitionFeatures">Current competition features.</param>
/// <returns>Frozen competition features.</returns>
protected virtual CompetitionFeatures CompleteFeatures(
[NotNull] CompetitionFeatures competitionFeatures) =>
competitionFeatures.Freeze();
public CompetitionState Run <T>(
[NotNull] T thisReference,
[CanBeNull] CompetitionFeatures competitionFeatures = null)
where T : class =>
RunCore(thisReference.GetType(), null, competitionFeatures);
public static IReadOnlyDictionary <Type, CompetitionState> Run(
[NotNull] Assembly assembly,
[CanBeNull] CompetitionFeatures competitionFeatures = null) =>
Runner.Run(assembly, competitionFeatures);
public static CompetitionState Run <T>(
[NotNull] T thisReference,
[CanBeNull] CompetitionFeatures competitionFeatures = null)
where T : class =>
Runner.Run(thisReference, competitionFeatures);
public static ICompetitionConfig CreateConfig(
[CanBeNull] Assembly targetAssembly = null,
[CanBeNull] CompetitionFeatures competitionFeatures = null) =>
CompetitionConfigFactory.FindFactoryAndCreate(targetAssembly, competitionFeatures);
public static CompetitionState Run <T>([CanBeNull] CompetitionFeatures competitionFeatures = null)
where T : class =>
Runner.Run <T>(competitionFeatures);
public static IReadOnlyDictionary <Type, CompetitionState> Run(
[NotNull] Type[] benchmarkTypes,
[CanBeNull] CompetitionFeatures competitionFeatures = null) =>
Runner.Run(benchmarkTypes, competitionFeatures);
