public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
        {
            var orderers =
                validationParameters
                .Benchmarks
                .Where(benchmark => benchmark.Config.Orderer != Order.DefaultOrderer.Instance)
                .Select(benchmark => benchmark.Config.Orderer)
                .Distinct();

            if (orderers.Count() > 1)
            {
                yield return(new ValidationError(true, "You use JoinSummary options, but provided configurations cannot be joined. Only one Orderer per benchmark cases is allowed."));
            }

            var styles =
                validationParameters
                .Benchmarks
                .Where(benchmark => benchmark.Config.SummaryStyle != SummaryStyle.Default &&
                       benchmark.Config.SummaryStyle != null)        // Paranoid
                .Select(benchmark => benchmark.Config.SummaryStyle)
                .Distinct();

            if (styles.Count() > 1)
            {
                yield return(new ValidationError(true, "You use JoinSummary options, but provided configurations cannot be joined. Only one SummaryStyle per benchmark cases is allowed."));
            }
        }
        public bool TreatsWarningsAsErrors => true; // it is a must!

        public IEnumerable <ValidationError> Validate(ValidationParameters input)
        {
            var allBenchmarks = input.Benchmarks.ToImmutableArray();
            var orderProvider = input.Config.Orderer;

            var benchmarkLogicalGroups = allBenchmarks
                                         .Select(benchmark => orderProvider.GetLogicalGroupKey(allBenchmarks, benchmark))
                                         .ToArray();

            var logicalGroups = benchmarkLogicalGroups.Distinct().ToArray();

            foreach (string logicalGroup in logicalGroups)
            {
                var    benchmarks          = allBenchmarks.Where((benchmark, index) => benchmarkLogicalGroups[index] == logicalGroup).ToArray();
                int    methodBaselineCount = benchmarks.Select(b => b.Descriptor).Distinct().Count(it => it.Baseline);
                int    jobBaselineCount    = benchmarks.Select(b => b.Job).Distinct().Count(it => it.Meta.Baseline);
                string className           = benchmarks.First().Descriptor.Type.Name;

                if (methodBaselineCount > 1)
                {
                    yield return(CreateError("benchmark method", "Baseline = true", logicalGroup, className, methodBaselineCount.ToString()));
                }

                if (jobBaselineCount > 1)
                {
                    yield return(CreateError("job", "Baseline = true", logicalGroup, className, jobBaselineCount.ToString()));
                }
            }
        }
        public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
        {
            if (validationParameters.Config.GetLoggers().IsEmpty())
            {
                const string errorMessage = "No loggers defined, you will not see any progress!";

                ConsoleLogger.Default.WriteLineError(errorMessage); // no loggers defined, so we need to somehow display this information

                yield return(new ValidationError(false, errorMessage));
            }

            if (validationParameters.Config.GetExporters().IsEmpty())
            {
                yield return(new ValidationError(false, "No exporters defined, results will not be persisted."));
            }
            if (validationParameters.Config.GetColumnProviders().IsEmpty())
            {
                yield return(new ValidationError(false, "No column providers defined, result table will be empty."));
            }

            var pathValidation = ValidateArtifactsPath(validationParameters.Config.ArtifactsPath);

            if (pathValidation != null)
            {
                yield return(pathValidation);
            }
        }
Exemple #4
0
        public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
        {
            foreach (var group in validationParameters.Benchmarks.GroupBy(benchmark => benchmark.Descriptor.Type.GetTypeInfo().Assembly))
            {
                foreach (var referencedAssemblyName in group.Key.GetReferencedAssemblies())
                {
                    var referencedAssembly = Assembly.Load(referencedAssemblyName);

                    // LINQPad.exe is non-optimized on purpose, see https://github.com/dotnet/BenchmarkDotNet/issues/580#issuecomment-345484889 for more details
                    // we don't warn about non-optimized dependency to LINQPad
                    // but we give extra hint if the dll with benchmark itself was build without optimization by LINQPad
                    if (referencedAssembly.IsJitOptimizationDisabled().IsTrue() && !referencedAssembly.IsLinqPad())
                    {
                        yield return(new ValidationError(
                                         TreatsWarningsAsErrors,
                                         $"Assembly {group.Key.GetName().Name} which defines benchmarks references non-optimized {referencedAssemblyName.Name}"
                                         + (TreatsWarningsAsErrors
                                ? $"{Environment.NewLine}\tIf you own this dependency, please, build it in RELEASE." +
                                            $"{Environment.NewLine}\tIf you don't, you can create custom config with {nameof(JitOptimizationsValidator)}.{nameof(DontFailOnError)} to disable our custom policy and allow this benchmark to run."
                                : string.Empty)));
                    }
                }

                if (group.Key.IsJitOptimizationDisabled().IsTrue())
                {
                    yield return(new ValidationError(
                                     TreatsWarningsAsErrors,
                                     $"Assembly {group.Key.GetName().Name} which defines benchmarks is non-optimized" + Environment.NewLine +
                                     "Benchmark was built without optimization enabled (most probably a DEBUG configuration). Please, build it in RELEASE."
                                     + (group.Key.IsLinqPad()
                            ? Environment.NewLine + "Please enable optimizations in your LINQPad. Go to Preferences -> Query and select \"compile with /optimize+\""
                            : string.Empty)));
                }
            }
        }
        public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
        {
            var errors = new List<ValidationError>();

            foreach (var typeGroup in validationParameters.Benchmarks.GroupBy(benchmark => benchmark.Target.Type))
            {
                object benchmarkTypeInstance = null;
                if (!TryCreateBenchmarkTypeInstance(typeGroup.Key, errors, out benchmarkTypeInstance))
                {
                    continue;
                }

                if (!TryToSetParamsFields(benchmarkTypeInstance, errors))
                {
                    continue;
                }

                if (!TryToSetParamsProperties(benchmarkTypeInstance, errors))
                {
                    continue;
                }


                if (!TryToCallSetup(benchmarkTypeInstance, errors))
                {
                    continue;
                }

                ExecuteBenchmarks(benchmarkTypeInstance, typeGroup, errors);
            }

            return errors;
        }
Exemple #6
0
        public bool TreatsWarningsAsErrors => true; // it is a must!

        public IEnumerable <ValidationError> Validate(ValidationParameters input)
        {
            var allBenchmarks = input.Benchmarks.ToArray();
            var orderProvider = input.Config.GetOrderer() ?? DefaultOrderer.Instance;

            var benchmarkLogicalGroups = allBenchmarks
                                         .Select(benchmark => orderProvider.GetLogicalGroupKey(input.Config, allBenchmarks, benchmark))
                                         .ToArray();

            var logicalGroups = benchmarkLogicalGroups.Distinct().ToArray();

            foreach (var logicalGroup in logicalGroups)
            {
                var benchmarks          = allBenchmarks.Where((benchmark, index) => benchmarkLogicalGroups[index] == logicalGroup).ToArray();
                var methodBaselineCount = benchmarks.Count(b => b.Descriptor.Baseline);
                var jobBaselineCount    = benchmarks.Count(b => b.Job.Meta.Baseline);
                var className           = benchmarks.First().Descriptor.Type.Name;

                if (methodBaselineCount > 1)
                {
                    yield return(CreateError("benchmark method", "Baseline = true", logicalGroup, className, methodBaselineCount.ToString()));
                }

                if (jobBaselineCount > 1)
                {
                    yield return(CreateError("job", "Baseline = true", logicalGroup, className, jobBaselineCount.ToString()));
                }

                if (methodBaselineCount > 0 && jobBaselineCount > 1)
                {
                    yield return(CreateError("job-benchmark pair", "Baseline property", logicalGroup, className, "both"));
                }
            }
        }
Exemple #7
0
        public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
        {
            var resolver = new CompositeResolver(EnvironmentResolver.Instance, EngineResolver.Instance); // TODO: use specified resolver.

            foreach (var benchmark in validationParameters.Benchmarks)
            {
                var run          = benchmark.Job.Run;
                int unrollFactor = run.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver);
                if (unrollFactor <= 0)
                {
                    yield return(new ValidationError(true, $"Specified UnrollFactor ({unrollFactor}) must be greater than zero", benchmark));
                }
                else if (run.HasValue(RunMode.InvocationCountCharacteristic))
                {
                    int invocationCount = run.InvocationCount;
                    if (invocationCount % unrollFactor != 0)
                    {
                        string message = $"Specified InvocationCount ({invocationCount}) must be a multiple of UnrollFactor ({unrollFactor})";
                        yield return(new ValidationError(true, message, benchmark));
                    }
                }

                foreach (var validationError in ValidateMinMax(run, resolver, benchmark, RunMode.MinIterationCountCharacteristic, RunMode.MaxIterationCountCharacteristic))
                {
                    yield return(validationError);
                }

                foreach (var validationError in ValidateMinMax(run, resolver, benchmark, RunMode.MinWarmupIterationCountCharacteristic, RunMode.MaxWarmupIterationCountCharacteristic))
                {
                    yield return(validationError);
                }
            }
        }
        public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
        {
            var errors = new List <ValidationError>();

            foreach (var typeGroup in validationParameters.Benchmarks.GroupBy(benchmark => benchmark.Descriptor.Type))
            {
                if (!TryCreateBenchmarkTypeInstance(typeGroup.Key, errors, out var benchmarkTypeInstance))
                {
                    continue;
                }

                if (!TryToSetParamsFields(benchmarkTypeInstance, errors))
                {
                    continue;
                }

                if (!TryToSetParamsProperties(benchmarkTypeInstance, errors))
                {
                    continue;
                }

                if (!TryToCallGlobalSetup(benchmarkTypeInstance, errors))
                {
                    continue;
                }

                ExecuteBenchmarks(benchmarkTypeInstance, typeGroup, errors);
            }

            return(errors);
        }
        public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
        {
            foreach (var group in validationParameters.Benchmarks.GroupBy(benchmark => benchmark.Descriptor.Type.GetTypeInfo().Assembly))
            {
                foreach (var referencedAssemblyName in group.Key.GetReferencedAssemblies())
                {
                    var referencedAssembly = Assembly.Load(referencedAssemblyName);

                    if (referencedAssembly.IsJitOptimizationDisabled().IsTrue() && !IsLinqPad(referencedAssembly))
                    {
                        yield return(new ValidationError(
                                         TreatsWarningsAsErrors,
                                         $"Assembly {group.Key.GetName().Name} which defines benchmarks references non-optimized {referencedAssemblyName.Name}"
                                         + (TreatsWarningsAsErrors
                                ? $"{Environment.NewLine}\tIf you own this dependency, please, build it in RELEASE." +
                                            $"{Environment.NewLine}\tIf you don't, you can create custom config with {nameof(DontFailOnError)} to disable our custom policy and allow this benchmark to run."
                                : string.Empty)));
                    }
                }

                if (group.Key.IsJitOptimizationDisabled().IsTrue())
                {
                    yield return(new ValidationError(
                                     TreatsWarningsAsErrors,
                                     $"Assembly {group.Key.GetName().Name} which defines benchmarks is non-optimized" + Environment.NewLine +
                                     "Benchmark was built without optimization enabled (most probably a DEBUG configuration). Please, build it in RELEASE."
                                     + (IsLinqPad(group.Key)
                            ? Environment.NewLine + "Please enable optimizations in your LINQPad. Go to Preferences -> Query and select \"compile with /optimize+\""
                            : string.Empty)));
                }
            }
        }
Exemple #10
0
 public IEnumerable <ValidationError> Validate(ValidationParameters input) =>
 input.Benchmarks
 .Select(benchmark => benchmark.Descriptor.Type)
 .Distinct()
 .SelectMany(type => type.GetTypeMembersWithGivenAttribute <ParamsAllValuesAttribute>(ReflectionFlags))
 .Distinct()
 .Select(member => GetErrorOrDefault(member.ParameterType))
 .Where(error => error != null);
Exemple #11
0
 public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
 => validationParameters
 .Benchmarks
 .Select(benchmark => benchmark.Descriptor.Type.Assembly)
 .Distinct()
 .SelectMany(assembly => assembly.GetRunnableBenchmarks())
 .SelectMany(GenericBenchmarksBuilder.BuildGenericsIfNeeded)
 .Where(result => !result.isSuccess)
 .Select(result => new ValidationError(false, $"Generic type {result.result.Name} failed to build due to wrong type argument or arguments count, ignoring."));
 public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
 => validationParameters
 .Benchmarks
 .Select(benchmark => benchmark.Target.Type.Assembly)
 .Distinct()
 .Where(assembly => assembly.Location.StartsWith(Path.GetTempPath()))
 .Select(
     assembly => new ValidationError(
         false,
         $"Assembly {assembly} is located in temp. If you are running benchmarks from xUnit you need to disable shadow copy. It's not supported by design."));
 public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
 => validationParameters
 .Benchmarks
 .Where(benchmark => !IsValidCSharpIdentifier(benchmark.Target.Method.Name))
 .Distinct(BenchmarkMethodEqualityComparer.Instance)         // we might have multiple jobs targeting same method. Single error should be enough ;)
 .Select(benchmark
         => new ValidationError(
             true,
             $"Benchmarked method `{benchmark.Target.Method.Name}` contains illegal character(s). Please use `[<Benchmark(Description = \"Custom name\")>]` to set custom display name.",
             benchmark
             ));
        public bool TreatsWarningsAsErrors => true; // it is a must!

        public IEnumerable<ValidationError> Validate(ValidationParameters input)
        {
            foreach (var groupByType in input.Benchmarks.GroupBy(benchmark => benchmark.Target.Type))
            {
                var allMethods = groupByType.Key.GetAllMethods();
                var count = allMethods.Count(method => method.GetCustomAttributes(false).OfType<BenchmarkAttribute>()
                                                             .Any(benchmarkAttribute => benchmarkAttribute.Baseline));
                if (count > 1)
                {
                    yield return new ValidationError(
                        TreatsWarningsAsErrors,
                        $"Only 1 [Benchmark] in a class can have \"Baseline = true\" applied to it, class {groupByType.Key.Name} has {count}");
                }
            }
        }
Exemple #15
0
        public bool TreatsWarningsAsErrors => true; // it is a must!

        public IEnumerable <ValidationError> Validate(ValidationParameters input)
        {
            foreach (var groupByType in input.Benchmarks.GroupBy(benchmark => benchmark.Target.Type))
            {
                var allMethods = groupByType.Key.GetAllMethods();
                var count      = allMethods.Count(method => method.GetCustomAttributes(false).OfType <BenchmarkAttribute>()
                                                  .Any(benchmarkAttribute => benchmarkAttribute.Baseline));
                if (count > 1)
                {
                    yield return(new ValidationError(
                                     TreatsWarningsAsErrors,
                                     $"Only 1 [Benchmark] in a class can have \"Baseline = true\" applied to it, class {groupByType.Key.Name} has {count}"));
                }
            }
        }
        public bool TreatsWarningsAsErrors => true; // it is a must!

        public IEnumerable <ValidationError> Validate(ValidationParameters input)
        {
            var validationErrors = new List <ValidationError>();

            foreach (var groupByType in input.Benchmarks.GroupBy(benchmark => benchmark.Descriptor.Type))
            {
                var allMethods = groupByType.Key.GetAllMethods().ToArray();

                validationErrors.AddRange(ValidateAttributes <GlobalSetupAttribute>(groupByType.Key.Name, allMethods));
                validationErrors.AddRange(ValidateAttributes <GlobalCleanupAttribute>(groupByType.Key.Name, allMethods));
                validationErrors.AddRange(ValidateAttributes <IterationSetupAttribute>(groupByType.Key.Name, allMethods));
                validationErrors.AddRange(ValidateAttributes <IterationSetupAttribute>(groupByType.Key.Name, allMethods));
            }

            return(validationErrors);
        }
        public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
        {
            var errors = new List <ValidationError>();

            foreach (var typeGroup in validationParameters.Benchmarks.GroupBy(benchmark => benchmark.Target.Type))
            {
                if (!TryCreateBenchmarkTypeInstance(typeGroup.Key, errors, out var benchmarkTypeInstance))
                {
                    continue;
                }

                ExecuteBenchmarks(benchmarkTypeInstance, typeGroup, errors);
            }

            return(errors);
        }
        public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
        {
            var resolver = new CompositeResolver(EnvResolver.Instance, EngineResolver.Instance); // TODO: use specified resolver.

            foreach (var benchmark in validationParameters.Benchmarks)
            {
                var run          = benchmark.Job.Run;
                int unrollFactor = run.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver);
                if (unrollFactor <= 0)
                {
                    yield return(new ValidationError(true, $"Specified UnrollFactor ({unrollFactor}) must be greater than zero", benchmark));
                }
                else if (run.HasValue(RunMode.InvocationCountCharacteristic))
                {
                    int invocationCount = run.InvocationCount;
                    if (invocationCount % unrollFactor != 0)
                    {
                        string message = $"Specified InvocationCount ({invocationCount}) must be a multiple of UnrollFactor ({unrollFactor})";
                        yield return(new ValidationError(true, message, benchmark));
                    }
                }

                int minTargetCount = run.ResolveValue(RunMode.MinTargetIterationCountCharacteristic, resolver);
                int maxTargetCount = run.ResolveValue(RunMode.MaxTargetIterationCountCharacteristic, resolver);

                if (minTargetCount <= 0)
                {
                    yield return(new ValidationError(true, $"{nameof(RunMode.MinTargetIterationCount)} must be greater than zero (was {minTargetCount})", benchmark));
                }

                if (maxTargetCount <= 0)
                {
                    yield return(new ValidationError(true, $"{nameof(RunMode.MaxTargetIterationCount)} must be greater than zero (was {maxTargetCount})", benchmark));
                }

                if (minTargetCount >= maxTargetCount)
                {
                    yield return(new ValidationError(true, $"{nameof(RunMode.MaxTargetIterationCount)} must be greater than {nameof(RunMode.MinTargetIterationCount)} (was {maxTargetCount} and {minTargetCount})", benchmark));
                }
            }
        }
 public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
 {
     var resolver = EnvResolver.Instance; // TODO: use specified resolver.
     foreach (var benchmark in validationParameters.Benchmarks)
     {
         var run = benchmark.Job.Run;
         int unrollFactor = run.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver);
         if (unrollFactor <= 0)
         {
             string message = $"Specified UnrollFactor ({unrollFactor}) must be greater than zero";
             yield return new ValidationError(true, message, benchmark);
         }
         else if (run.HasValue(RunMode.InvocationCountCharacteristic))
         {
             int invocationCount = run.InvocationCount;
             if (invocationCount % unrollFactor != 0)
             {
                 string message = $"Specified InvocationCount ({invocationCount}) must be a multiple of UnrollFactor ({unrollFactor})";
                 yield return new ValidationError(true, message, benchmark);
             }
         }
     }
 }
        public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
        {
            var resolver = EnvResolver.Instance; // TODO: use specified resolver.

            foreach (var benchmark in validationParameters.Benchmarks)
            {
                var run          = benchmark.Job.Run;
                int unrollFactor = run.ResolveValue(RunMode.UnrollFactorCharacteristic, resolver);
                if (unrollFactor <= 0)
                {
                    string message = $"Specified UnrollFactor ({unrollFactor}) must be greater than zero";
                    yield return(new ValidationError(true, message, benchmark));
                }
                else if (run.HasValue(RunMode.InvocationCountCharacteristic))
                {
                    int invocationCount = run.InvocationCount;
                    if (invocationCount % unrollFactor != 0)
                    {
                        string message = $"Specified InvocationCount ({invocationCount}) must be a multiple of UnrollFactor ({unrollFactor})";
                        yield return(new ValidationError(true, message, benchmark));
                    }
                }
            }
        }
Exemple #21
0
        public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
        {
            foreach (var group in validationParameters.Benchmarks.GroupBy(benchmark => benchmark.Target.Type.GetTypeInfo().Assembly))
            {
                foreach (var referencedAssemblyName in group.Key.GetReferencedAssemblies())
                {
                    var referencedAssembly = Assembly.Load(referencedAssemblyName);

                    if (referencedAssembly.IsJitOptimizationDisabled().IsTrue())
                    {
                        yield return(new ValidationError(
                                         TreatsWarningsAsErrors,
                                         $"Assembly {group.Key.GetName().Name} which defines benchmarks references non-optimized {referencedAssemblyName.Name}"));
                    }
                }

                if (group.Key.IsJitOptimizationDisabled().IsTrue())
                {
                    yield return(new ValidationError(
                                     TreatsWarningsAsErrors,
                                     $"Assembly {group.Key.GetName().Name} which defines benchmarks is non-optimized"));
                }
            }
        }
 public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
 => validationParameters
 .Config
 .GetDiagnosers()
 .SelectMany(diagnoser => diagnoser.Validate(validationParameters));
Exemple #23
0
 public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
 => ValidateCSharpNaming(validationParameters.Benchmarks)
 .Union(ValidateNamingConflicts(validationParameters.Benchmarks))
 .Union(ValidateAccessModifiers(validationParameters.Benchmarks));
 public IEnumerable<ValidationError> Validate(ValidationParameters input)
 {
     yield return new ValidationError(true, "It just fails");
 }
Exemple #25
0
 public IEnumerable <ValidationError> Validate(ValidationParameters validationParameters)
 {
     return(Validators.SelectMany(validator => validator.Validate(validationParameters)));
 }
 public IEnumerable<ValidationError> Validate(ValidationParameters validationParameters)
 {
     return Validators.SelectMany(validator => validator.Validate(validationParameters));
 }