コード例 #1
0
ファイル: CompetitionRunnerBase.cs プロジェクト: ili/CodeJam
        private void FixConfigLoggers(ManualCompetitionConfig competitionConfig)
        {
            var runOptions  = competitionConfig.Options.RunOptions;
            var hostLogMode = runOptions.DetailedLogging ? LogFilter.AllMessages : LogFilter.PrefixedOrErrors;

            competitionConfig.Loggers.Insert(0, CreateHostLogger(hostLogMode));
        }
コード例 #2
0
        public static void TestInProcessBenchmarkWithValidation()
        {
            ResetCounters();

            // DONTTOUCH: config SHOULD NOT match the default platform (x64).
            var config = new ManualCompetitionConfig(
                CompetitionHelpers.CreateConfig(
                    typeof(InProcessBenchmarkAllCases),
                    new CompetitionFeatures
            {
                Platform = Platform.X86
            }));

            config.Add(InProcessValidator.FailOnError);

            var summary = SelfTestCompetition
                          .Run <InProcessBenchmarkAllCases>(config)
                          .LastRunSummary;

            Assert.AreEqual(_callCounter, 0);
            Assert.AreEqual(_afterSetupCounter, 0);
            Assert.AreEqual(_setupCounter, 0);
            Assert.AreEqual(_cleanupCounter, 0);

            Assert.AreEqual(summary?.ValidationErrors.Length, 1);
            Assert.IsTrue(summary?.ValidationErrors[0].IsCritical);
            Assert.AreEqual(
                summary?.ValidationErrors[0].Message,
                "Job SelfTestConfigX86, EnvMode.Platform was run as X64 (X86 expected). Fix your test runner options.");
        }
コード例 #3
0
 public void Modify(ManualCompetitionConfig competitionConfig)
 {
     competitionConfig.Metrics.Add(
         WellKnownEtwMetrics.FileIoRead);
     competitionConfig.Metrics.Add(
         WellKnownEtwMetrics.FileIoWrite);
 }
コード例 #4
0
ファイル: CompetitionRunnerBase.cs プロジェクト: ili/CodeJam
        private static void FixConfigValidators(ManualCompetitionConfig competitionConfig)
        {
            var competitionOptions = competitionConfig.Options;
            var debugMode          = competitionOptions.RunOptions.AllowDebugBuilds;

            var validators      = competitionConfig.Validators;
            var customToolchain = competitionConfig.Jobs.Any(j => !(j.Infrastructure.Toolchain is InProcessToolchain));

            if (!customToolchain &&
                !validators.Any(v => v is InProcessValidator))
            {
                validators.Insert(0, debugMode ? InProcessValidator.DontFailOnError : InProcessValidator.FailOnError);
            }

            if (debugMode)
            {
                validators.RemoveAll(v => v is JitOptimizationsValidator);
            }
            else if (competitionOptions.Adjustments.AdjustMetrics &&
                     !validators.OfType <JitOptimizationsValidator>().Any())
            {
                validators.Insert(0, JitOptimizationsValidator.FailOnError);
            }

            Code.BugIf(
                validators.OfType <RunStateSlots>().Any(),
                "The config validators should not contain instances of RunStateSlots.");

            // DONTTOUCH: the RunStateSlots should be first in the chain.
            validators.Insert(0, new RunStateSlots());
        }
コード例 #5
0
 private void FixConfigExporters(ManualCompetitionConfig competitionConfig)
 {
     // HACK: shuts up the ConfigValidator
     if (competitionConfig.Exporters.Count == 0)
     {
         competitionConfig.Exporters.Add(new StubExporter());
     }
 }
コード例 #6
0
ファイル: CompetitionRunnerBase.cs プロジェクト: ili/CodeJam
 private void FixCompetitionConfig(ManualCompetitionConfig competitionConfig)
 {
     // DONTTOUCH: call order is important.
     FixConfigJobs(competitionConfig);
     FixConfigLoggers(competitionConfig);
     FixConfigValidators(competitionConfig);
     FixConfigAnalysers(competitionConfig);
     FixConfigMetrics(competitionConfig);
     FixConfigDuplicates(competitionConfig);
 }
コード例 #7
0
        public void RunSensitivityPerfTests()
        {
            // The test could fail with "too fast" warning, it's ok
            var overrideConfig = new ManualCompetitionConfig(RunConfig)
            {
                ReportWarningsAsErrors = false
            };

            Competition.Run(this, overrideConfig);
        }
コード例 #8
0
ファイル: CompetitionRunnerBase.cs プロジェクト: ili/CodeJam
        private static void FixConfigMetrics(ManualCompetitionConfig competitionConfig)
        {
            var metrics = competitionConfig.Metrics;

            competitionConfig.Add(
                metrics.Select(m => m.GetColumnProvider()).Where(c => c != null).ToArray());

            competitionConfig.Add(
                metrics.Select(m => m.GetDiagnosers()).SelectMany(d => d).ToArray());
        }
コード例 #9
0
 /// <summary>Updates competition config.</summary>
 /// <param name="competitionConfig">The competition config.</param>
 public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.ApplyModifier(
     new Job
 {
     Run =
     {
         LaunchCount =   1,
         WarmupCount = 256,
         TargetCount = 512
     }
 });
コード例 #10
0
        public void RunOpsCountNotSensitivePerfTests()
        {
            // The test should fail with warning!
            var overrideConfig = new ManualCompetitionConfig(RunConfig)
            {
                ReportWarningsAsErrors = false
            };

            Competition.Run(this, overrideConfig);
        }
コード例 #11
0
ファイル: CompetitionRunnerBase.cs プロジェクト: ili/CodeJam
 private static void FixConfigDuplicates(ManualCompetitionConfig competitionConfig)
 {
     RemoveDuplicates(competitionConfig.Analysers);
     RemoveDuplicates(competitionConfig.ColumnProviders);
     RemoveDuplicates(competitionConfig.Diagnosers);
     RemoveDuplicates(competitionConfig.Exporters);
     RemoveDuplicates(competitionConfig.Jobs);
     RemoveDuplicates(competitionConfig.Loggers);
     RemoveDuplicates(competitionConfig.Metrics, m => m.AttributeType);
     RemoveDuplicates(competitionConfig.Validators);
 }
コード例 #12
0
        private static ManualCompetitionConfig CreateRunConfigCore()
        {
            var result = new ManualCompetitionConfig
            {
                RerunIfLimitsFailed    = true,
                ReportWarningsAsErrors = true
            };

            result.Add(BenchmarkDotNet.Configs.DefaultConfig.Instance.GetColumns().ToArray());
            result.Add(AppConfigHelpers.GetImportantOnlyLogger(typeof(SelfTestHelpers).Assembly));
            return(result);
        }
コード例 #13
0
 /// <summary>Updates competition config.</summary>
 /// <param name="competitionConfig">The competition config.</param>
 public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.ApplyModifier(
     new Job
 {
     Run =
     {
         LaunchCount     =  1,
         WarmupCount     = 10,
         TargetCount     = 10,
         InvocationCount =  1,
         UnrollFactor    = 1
     }
 });
コード例 #14
0
ファイル: CompetitionRunnerBase.cs プロジェクト: ili/CodeJam
        private static void FixConfigAnalysers(ManualCompetitionConfig competitionConfig)
        {
            Code.BugIf(
                competitionConfig.Analysers.OfType <CompetitionAnalyser>().Any(),
                "The config analysers should not contain instances of CompetitionAnalyser.");

            // DONTTOUCH: these should be first analysers in the chain.
            competitionConfig.Analysers.Insert(0, ValidatorMessagesAnalyser.Instance);
            competitionConfig.Analysers.Insert(0, CompetitionPreconditionsAnalyser.Instance);

            // DONTTOUCH: the CompetitionAnnotateAnalyser should be last analyser in the chain.
            competitionConfig.Analysers.Add(CompetitionAnalyser.Instance);
        }
コード例 #15
0
 /// <summary>Applies <see cref="ICompetitionModifierSource"/> attribute modifiers.</summary>
 /// <param name="competitionConfig">The competition configuration.</param>
 /// <param name="metadataSource">The metadata source.</param>
 protected virtual void ApplyCompetitionModifiers(
     [NotNull] ManualCompetitionConfig competitionConfig,
     [CanBeNull] ICustomAttributeProvider metadataSource)
 {
     if (metadataSource != null)
     {
         foreach (var modifierSource in metadataSource
                  .GetMetadataAttributes <ICompetitionModifierSource>()
                  .Reverse())
         {
             modifierSource.Modifier.Modify(competitionConfig);
         }
     }
 }
コード例 #16
0
 public void Modify(ManualCompetitionConfig competitionConfig) =>
 competitionConfig.ApplyModifier(
     new CompetitionOptions()
 {
     Adjustments =
     {
         AdjustMetrics = false
     },
     Annotations =
     {
         IgnoreExistingAnnotations = true,
         PreviousRunLogUri         = _previousRunLogUri
     }
 });
コード例 #17
0
        protected virtual ManualCompetitionConfig CreateEmptyConfig(
            [CanBeNull] ICustomAttributeProvider metadataSource,
            [NotNull] CompetitionFeatures competitionFeatures)
        {
            var result = new ManualCompetitionConfig(BenchmarkDotNet.Configs.DefaultConfig.Instance);

            // DONTTOUCH: competition should not use default
            // Jobs, Diagnosers, Exporters and Loggers.
            // These omitted intentionally.
            result.Jobs.Clear();
            result.Diagnosers.Clear();
            result.Exporters.Clear();
            result.Loggers.Clear();

            // TODO: better columns.
            // TODO: columns for options.
            result.ColumnProviders.Clear();
            result.Add(competitionFeatures.TroubleshootingMode ? TroubleshootingModeColumns : DefaultColumns);

            var targetAssembly = GetAssembly(metadataSource);

            if (targetAssembly != null)
            {
                if (competitionFeatures.TroubleshootingMode)
                {
                    result.Add(
                        GetImportantInfoLogger(targetAssembly),
                        GetDetailedLogger(targetAssembly));

                    result.Add(Exporters.CsvTimingsExporter.Default);
                }
                else
                {
                    if (competitionFeatures.ImportantInfoLogger || competitionFeatures.ContinuousIntegrationMode)
                    {
                        result.Add(GetImportantInfoLogger(targetAssembly));
                    }
                    if (competitionFeatures.DetailedLogger)
                    {
                        result.Add(GetDetailedLogger(targetAssembly));
                    }
                }
            }

            result.Add(DefaultMetrics.ToArray());

            return(result);
        }
コード例 #18
0
ファイル: CompetitionRunnerBase.cs プロジェクト: ili/CodeJam
        private ICompetitionConfig CreateBenchmarkConfig(
            [NotNull] Type benchmarkType,
            ICompetitionConfig competitionConfig,
            CompetitionFeatures competitionFeatures)
        {
            // ReSharper disable once UseObjectOrCollectionInitializer
            var result = new ManualCompetitionConfig(
                competitionConfig ??
                CompetitionConfigFactory.FindFactoryAndCreate(benchmarkType, competitionFeatures));

            InitCompetitionConfigOverride(result);

            FixCompetitionConfig(result);

            return(result.AsReadOnly());
        }
コード例 #19
0
        public static void CompetitionTooSlowOk()
        {
            var overrideConfig = new ManualCompetitionConfig(SelfTestConfig)
            {
                AllowLongRunningBenchmarks = true
            };

            var runState = SelfTestCompetition.Run <TooSlowBenchmark>(overrideConfig);
            var messages = runState.GetMessages();
            var summary  = runState.LastRunSummary;

            Assert.AreEqual(summary?.ValidationErrors.Length, 0);
            Assert.AreEqual(runState.RunNumber, 1);
            Assert.AreEqual(runState.RunsLeft, 0);
            Assert.AreEqual(runState.RunLimitExceeded, false);
            Assert.AreEqual(runState.LooksLikeLastRun, true);
            Assert.AreEqual(messages.Length, 0);
        }
コード例 #20
0
        /// <summary>Applies <see cref="IConfigSource"/> attribute modifiers.</summary>
        /// <param name="competitionConfig">The competition configuration.</param>
        /// <param name="metadataSource">The metadata source.</param>
        protected virtual void ApplyConfigSources(ManualCompetitionConfig competitionConfig, ICustomAttributeProvider metadataSource)
        {
            var config = new ManualConfig();
            var add    = false;

            if (metadataSource != null)
            {
                foreach (var modifierSource in metadataSource
                         .GetMetadataAttributes <IConfigSource>()
                         .Reverse())
                {
                    config = ManualConfig.Union(config, modifierSource.Config);
                    add    = true;
                }
            }
            if (add)
            {
                competitionConfig.Add(config);
            }
        }
コード例 #21
0
        /// <summary>Helper for custom configs: configuration that should be used for new performance tests.</summary>
        /// <param name="job">The job for the config. Default one will be used if <c>null</c>.</param>
        /// <returns>Configuration that should be used for new performance tests.</returns>
        public static ManualCompetitionConfig CreateDefaultConfig(IJob job = null)
        {
            var defaultConfig = BenchmarkDotNet.Configs.DefaultConfig.Instance;

            var result = new ManualCompetitionConfig
            {
                RerunIfLimitsFailed = true,
                KeepBenchmarkFiles  = defaultConfig.KeepBenchmarkFiles
            };

            result.Add(defaultConfig.GetColumns().ToArray());
            result.Add(defaultConfig.GetValidators().ToArray());
            result.Add(defaultConfig.GetAnalysers().ToArray());
            result.Add(defaultConfig.GetDiagnosers().ToArray());
            result.Set(defaultConfig.GetOrderProvider());

            result.Add(job ?? CreateDefaultJob());

            return(result);
        }
コード例 #22
0
ファイル: CompetitionRunnerBase.cs プロジェクト: ili/CodeJam
        private static void FixConfigJobs(ManualCompetitionConfig competitionConfig)
        {
            var jobs = competitionConfig.Jobs;

            for (var i = 0; i < jobs.Count; i++)
            {
                var job = jobs[i];
                if (job.Infrastructure.Toolchain == null)
                {
                    var id = job.HasValue(CharacteristicObject.IdCharacteristic) ? job.Id : null;
                    jobs[i] = job
                              .With(InProcessToolchain.Instance)
                              .WithId(id);
                }
            }

            if (competitionConfig.Jobs.Count == 0)
            {
                competitionConfig.Add(CompetitionConfigFactory.DefaultJob);
            }
        }
コード例 #23
0
 public void Modify(ManualCompetitionConfig competitionConfig) =>
 competitionConfig.Add(StatisticColumn.Skewness);
コード例 #24
0
 public void Modify(ManualCompetitionConfig competitionConfig) =>
 competitionConfig.Add(BaselineScaledColumn.WelchTTestPValue);
コード例 #25
0
 public void Modify(ManualCompetitionConfig competitionConfig) =>
 competitionConfig.Add(StatisticColumn.Kurtosis);
コード例 #26
0
 /// <summary>Customize competition config.</summary>
 /// <param name="competitionConfig">The competition configuration.</param>
 protected override void InitCompetitionConfigOverride(ManualCompetitionConfig competitionConfig)
 {
     base.InitCompetitionConfigOverride(competitionConfig);
     competitionConfig.Loggers.RemoveAll(l => l is ConsoleLogger);
 }
コード例 #27
0
 /// <summary>Completes competition config creation.</summary>
 /// <param name="competitionConfig">Current competition config.</param>
 /// <returns>Read-only competition config.</returns>
 protected virtual ICompetitionConfig CompleteConfig(
     [NotNull] ManualCompetitionConfig competitionConfig) =>
 competitionConfig.AsReadOnly();
コード例 #28
0
 public void Modify(ManualCompetitionConfig competitionConfig) => competitionConfig.Metrics.Add(WellKnownEtwMetrics.ClrExceptions);
コード例 #29
0
 public void Modify(ManualCompetitionConfig competitionConfig)
 {
     competitionConfig.Metrics.Add(WellKnownMetrics.ExpectedTime);
     competitionConfig.Metrics.Add(WellKnownMetrics.GcAllocations);
 }
コード例 #30
0
ファイル: CompetitionRunnerBase.cs プロジェクト: ili/CodeJam
 /// <summary>Customize competition config.</summary>
 /// <param name="competitionConfig">The competition configuration.</param>
 protected virtual void InitCompetitionConfigOverride(ManualCompetitionConfig competitionConfig)
 {
 }