public BenchmarkTask(int processCount, BenchmarkConfiguration configuration, BenchmarkSettings settings, BenchmarkParams @params = null) { ProcessCount = processCount; Configuration = configuration; Settings = settings; Params = @params; }
private const long InvokeTimoutMilliseconds = 1000; // TODO: Move to settings #endregion Fields #region Methods public void SingleRun(BenchmarkSettings settings, long operationsPerInvoke, Action setupAction, Action targetAction, Action idleAction) { for (int i = 0; i < settings.WarmupIterationCount; i++) MultiInvoke("// Warmup " + (i + 1), setupAction, targetAction, 1, operationsPerInvoke); for (int i = 0; i < settings.TargetIterationCount; i++) MultiInvoke("Target " + (i + 1), setupAction, targetAction, 1, operationsPerInvoke); }
public static IEnumerable<BenchmarkTask> Resolve(MethodInfo methodInfo, BenchmarkSettings defaultSettings) { var attrs = methodInfo.GetCustomAttributes(typeof(TaskAttribute), false).Cast<TaskAttribute>().ToList(); if (attrs.Count == 0) attrs = methodInfo.DeclaringType.GetCustomAttributes(typeof(TaskAttribute), false).Cast<TaskAttribute>().ToList(); if (attrs.Count == 0) attrs.Add(new TaskAttribute(warmupIterationCount: defaultSettings.WarmupIterationCount, targetIterationCount: defaultSettings.TargetIterationCount)); return attrs.Select(attr => attr.Task); }
public static IEnumerable <BenchmarkTask> Resolve(MethodInfo methodInfo, BenchmarkSettings defaultSettings) { var attrs = methodInfo.GetCustomAttributes(typeof(BenchmarkTaskAttribute), false).Cast <BenchmarkTaskAttribute>().ToList(); if (attrs.Count == 0) { attrs = methodInfo.DeclaringType.GetCustomAttributes(typeof(BenchmarkTaskAttribute), false).Cast <BenchmarkTaskAttribute>().ToList(); } if (attrs.Count == 0) { attrs.Add(new BenchmarkTaskAttribute(warmupIterationCount: defaultSettings.WarmupIterationCount, targetIterationCount: defaultSettings.TargetIterationCount)); } return(attrs.Select(attr => attr.Task)); }
public void Throughput(BenchmarkSettings settings, long operationsPerInvoke, Action setupAction, Action targetAction, Action idleAction) { setupAction(); targetAction(); idleAction(); long invokeCount = 1; double lastPreWarmupMilliseconds = 0; while (true) { var measurement = MultiInvoke("// Pre-Warmup", setupAction, targetAction, invokeCount, operationsPerInvoke); lastPreWarmupMilliseconds = measurement.Milliseconds; if (lastPreWarmupMilliseconds > InvokeTimoutMilliseconds) break; if (lastPreWarmupMilliseconds < 1) invokeCount *= InvokeTimoutMilliseconds; else invokeCount *= (long)Math.Ceiling(InvokeTimoutMilliseconds / lastPreWarmupMilliseconds); } double idleMilliseconds = 0; for (int i = 0; i < Math.Min(3, settings.WarmupIterationCount); i++) { var measurement = MultiInvoke("// Warmup (idle)", setupAction, idleAction, invokeCount, operationsPerInvoke); idleMilliseconds = measurement.Milliseconds; } invokeCount = invokeCount * 1000 / (long)Math.Round(Math.Min(1000, Math.Max(100, lastPreWarmupMilliseconds - idleMilliseconds))); Console.WriteLine("// IterationCount = " + invokeCount); long idleTicks = 0; var targetIdleInvokeCount = Math.Min(5, settings.TargetIterationCount); for (int i = 0; i < targetIdleInvokeCount; i++) { var measurement = MultiInvoke("// Target (idle)", setupAction, idleAction, invokeCount, operationsPerInvoke); idleTicks += measurement.Ticks; } idleTicks /= targetIdleInvokeCount; for (int i = 0; i < settings.WarmupIterationCount; i++) MultiInvoke("// Warmup " + (i + 1), setupAction, targetAction, invokeCount, operationsPerInvoke, idleTicks); for (int i = 0; i < settings.TargetIterationCount; i++) MultiInvoke("Target " + (i + 1), setupAction, targetAction, invokeCount, operationsPerInvoke, idleTicks); }
private static IEnumerable<Benchmark> CompetitionToBenchmarks(object competition, BenchmarkSettings defaultSettings) { if (defaultSettings == null) defaultSettings = BenchmarkSettings.CreateDefault(); var targetType = competition.GetType(); var methods = targetType.GetMethods(); var setupMethod = methods.FirstOrDefault(m => m.ResolveAttribute<SetupAttribute>() != null); if (setupMethod != null) { // setupMethod is optional, but if it's there it must have the correct signature, accessibility, etc AssertMethodHasCorrectSignature("Setup", setupMethod); AssertMethodIsAccessible("Setup", setupMethod); AssertMethodIsNotDeclaredInGeneric("Setup", setupMethod); AssertMethodIsNotGeneric("Setup", setupMethod); } for (int i = 0; i < methods.Length; i++) { var methodInfo = methods[i]; var benchmarkAttribute = methodInfo.ResolveAttribute<BenchmarkAttribute>(); if (benchmarkAttribute != null) { var target = new BenchmarkTarget(targetType, methodInfo, setupMethod, benchmarkAttribute.Description); AssertMethodHasCorrectSignature("Benchmark", methodInfo); AssertMethodIsAccessible("Benchmark", methodInfo); AssertMethodIsNotDeclaredInGeneric("Benchmark", methodInfo); AssertMethodIsNotGeneric("Benchmark", methodInfo); foreach (var task in BenchmarkTask.Resolve(methodInfo, defaultSettings)) yield return new Benchmark(target, task); } } }
public IEnumerable<BenchmarkReport> RunUrl(string url, BenchmarkSettings defaultSettings = null) { return RunCompetition(UrlToBenchmarks(url, defaultSettings).ToList()); }
public IEnumerable<BenchmarkReport> RunSource(string source, BenchmarkSettings defaultSettings = null) { return RunCompetition(SourceToBenchmarks(source, defaultSettings).ToList()); }
public IEnumerable<BenchmarkReport> RunCompetition(object benchmarkCompetition, BenchmarkSettings defaultSettings = null) { return RunCompetition(CompetitionToBenchmarks(benchmarkCompetition, defaultSettings).ToList()); }
private static IEnumerable<Benchmark> UrlToBenchmarks(string url, BenchmarkSettings defaultSettings) { string benchmarkContent = String.Empty; try { var webRequest = WebRequest.Create(url); using (var response = webRequest.GetResponse()) using (var content = response.GetResponseStream()) using (var reader = new StreamReader(content)) benchmarkContent = reader.ReadToEnd(); if (string.IsNullOrWhiteSpace(benchmarkContent)) { Console.WriteLine($"content of '{url}' is empty."); yield break; } } catch (Exception e) { Console.WriteLine("Exception: " + e.Message); yield break; } var cSharpCodeProvider = new CSharpCodeProvider(); var compilerParameters = new CompilerParameters(new[] { "mscorlib.dll", "System.Core.dll" }) { CompilerOptions = "/unsafe" }; compilerParameters.ReferencedAssemblies.Add(typeof(BenchmarkRunner).Assembly.Location); var compilerResults = cSharpCodeProvider.CompileAssemblyFromSource(compilerParameters, benchmarkContent); if (compilerResults.Errors.HasErrors) { compilerResults.Errors.Cast<CompilerError>().ToList().ForEach(error => Console.WriteLine(error.ErrorText)); yield break; } foreach (var type in compilerResults.CompiledAssembly.GetTypes()) { var instance = Activator.CreateInstance(type); foreach (var benchmark in CompetitionToBenchmarks(instance, defaultSettings)) yield return new Benchmark(new BenchmarkTarget(benchmark.Target.Type, benchmark.Target.Method, benchmark.Target.Description, benchmarkContent), benchmark.Task); } }
private static IEnumerable<Benchmark> CompetitionToBenchmarks(object competition, BenchmarkSettings defaultSettings) { if (defaultSettings == null) defaultSettings = BenchmarkSettings.CreateDefault(); var targetType = competition.GetType(); var methods = targetType.GetMethods(); for (int i = 0; i < methods.Length; i++) { var methodInfo = methods[i]; var benchmarkAttribute = methodInfo.ResolveAttribute<BenchmarkAttribute>(); if (benchmarkAttribute != null) { var target = new BenchmarkTarget(targetType, methodInfo, benchmarkAttribute.Description); AssertBenchmarkMethodHasCorrectSignature(methodInfo); AssertBenchmarkMethodIsAccessible(methodInfo); AssertBenchmarkMethodIsNotDeclaredInGeneric(methodInfo); AssertBenchmarkMethodIsNotGeneric(methodInfo); foreach (var task in BenchmarkTask.Resolve(methodInfo, defaultSettings)) yield return new Benchmark(target, task); } } }
public BenchmarkTask(int processCount, BenchmarkConfiguration configuration, BenchmarkSettings settings) { ProcessCount = processCount; Configuration = configuration; Settings = settings; }
private static IEnumerable<Benchmark> CompetitionToBenchmarks(object competition, BenchmarkSettings defaultSettings) { if (defaultSettings == null) defaultSettings = BenchmarkSettings.CreateDefault(); var targetType = competition.GetType(); var methods = targetType.GetMethods(); var setupMethod = methods.FirstOrDefault(m => m.ResolveAttribute<SetupAttribute>() != null); if (setupMethod != null) { // setupMethod is optional, but if it's there it must have the correct signature, accessibility, etc AssertMethodHasCorrectSignature("Setup", setupMethod); AssertMethodIsAccessible("Setup", setupMethod); AssertMethodIsNotDeclaredInGeneric("Setup", setupMethod); AssertMethodIsNotGeneric("Setup", setupMethod); } // If there is one, get the single Field or Property that has the [Params(..)] attribute var fields = targetType.GetFields().Select(f => new { f.Name, Attribute = f.ResolveAttribute<ParamsAttribute>(), IsStatic = f.IsStatic, }); var properties = targetType.GetProperties().Select(f => new { f.Name, Attribute = f.ResolveAttribute<ParamsAttribute>(), IsStatic = f.GetSetMethod().IsStatic }); var fieldOrProperty = fields.Concat(properties).FirstOrDefault(i => i.Attribute != null); for (int i = 0; i < methods.Length; i++) { var methodInfo = methods[i]; var benchmarkAttribute = methodInfo.ResolveAttribute<BenchmarkAttribute>(); if (benchmarkAttribute != null) { var target = new BenchmarkTarget(targetType, methodInfo, setupMethod, benchmarkAttribute.Description); AssertMethodHasCorrectSignature("Benchmark", methodInfo); AssertMethodIsAccessible("Benchmark", methodInfo); AssertMethodIsNotDeclaredInGeneric("Benchmark", methodInfo); AssertMethodIsNotGeneric("Benchmark", methodInfo); foreach (var task in BenchmarkTask.Resolve(methodInfo, defaultSettings)) { if (fieldOrProperty == null) { yield return new Benchmark(target, task); } else { var @params = new BenchmarkParams(fieldOrProperty.Name, fieldOrProperty.IsStatic, fieldOrProperty.Attribute.Args); // All the properties of BenchmarkTask and it's children are immutable, so cloning a BenchmarkTask like this should be safe var newTask = new BenchmarkTask(task.ProcessCount, task.Configuration, task.Settings, @params); yield return new Benchmark(target, newTask); } } } } }
private static IEnumerable<Benchmark> SourceToBenchmarks(string source, BenchmarkSettings defaultSettings) { string benchmarkContent = source; var cSharpCodeProvider = new CSharpCodeProvider(); var compilerParameters = new CompilerParameters(new[] { "mscorlib.dll", "System.Core.dll" }) { CompilerOptions = "/unsafe" }; compilerParameters.ReferencedAssemblies.Add(typeof(BenchmarkRunner).Assembly.Location); var compilerResults = cSharpCodeProvider.CompileAssemblyFromSource(compilerParameters, benchmarkContent); if (compilerResults.Errors.HasErrors) { compilerResults.Errors.Cast<CompilerError>().ToList().ForEach(error => Console.WriteLine(error.ErrorText)); yield break; } foreach (var type in compilerResults.CompiledAssembly.GetTypes()) { var instance = Activator.CreateInstance(type); foreach (var benchmark in CompetitionToBenchmarks(instance, defaultSettings)) { yield return new Benchmark(new BenchmarkTarget(benchmark.Target.Type, benchmark.Target.Method, benchmark.Target.SetupMethod, benchmark.Target.Description, benchmarkContent), benchmark.Task); } } }
private static IEnumerable<Benchmark> UrlToBenchmarks(string url, BenchmarkSettings defaultSettings) { string benchmarkContent = String.Empty; try { var webRequest = WebRequest.Create(url); using (var response = webRequest.GetResponse()) using (var content = response.GetResponseStream()) using (var reader = new StreamReader(content)) benchmarkContent = reader.ReadToEnd(); if (string.IsNullOrWhiteSpace(benchmarkContent)) { Console.WriteLine($"content of '{url}' is empty."); return new Benchmark[0]; } } catch (Exception e) { Console.WriteLine("Exception: " + e.Message); return new Benchmark[0]; } return SourceToBenchmarks(benchmarkContent, defaultSettings); }
public void RunBenchmark(BenchmarkSettings settings) { new BenchmarkMethodInvoker().Throughput(settings, 1, setupAction, targetAction, idleAction); }