public BenchmarkTask(int processCount, BenchmarkConfiguration configuration, BenchmarkSettings settings, BenchmarkParams @params = null)
{
ProcessCount = processCount;
Configuration = configuration;
Settings = settings;
Params = @params;
}
private const long InvokeTimoutMilliseconds = 1000; // TODO: Move to settings
#endregion Fields
#region Methods
public void SingleRun(BenchmarkSettings settings, long operationsPerInvoke, Action setupAction, Action targetAction, Action idleAction)
{
for (int i = 0; i < settings.WarmupIterationCount; i++)
MultiInvoke("// Warmup " + (i + 1), setupAction, targetAction, 1, operationsPerInvoke);
for (int i = 0; i < settings.TargetIterationCount; i++)
MultiInvoke("Target " + (i + 1), setupAction, targetAction, 1, operationsPerInvoke);
}
public static IEnumerable<BenchmarkTask> Resolve(MethodInfo methodInfo, BenchmarkSettings defaultSettings)
{
var attrs = methodInfo.GetCustomAttributes(typeof(TaskAttribute), false).Cast<TaskAttribute>().ToList();
if (attrs.Count == 0)
attrs = methodInfo.DeclaringType.GetCustomAttributes(typeof(TaskAttribute), false).Cast<TaskAttribute>().ToList();
if (attrs.Count == 0)
attrs.Add(new TaskAttribute(warmupIterationCount: defaultSettings.WarmupIterationCount, targetIterationCount: defaultSettings.TargetIterationCount));
return attrs.Select(attr => attr.Task);
}
public static IEnumerable <BenchmarkTask> Resolve(MethodInfo methodInfo, BenchmarkSettings defaultSettings)
{
var attrs = methodInfo.GetCustomAttributes(typeof(BenchmarkTaskAttribute), false).Cast <BenchmarkTaskAttribute>().ToList();
if (attrs.Count == 0)
{
attrs = methodInfo.DeclaringType.GetCustomAttributes(typeof(BenchmarkTaskAttribute), false).Cast <BenchmarkTaskAttribute>().ToList();
}
if (attrs.Count == 0)
{
attrs.Add(new BenchmarkTaskAttribute(warmupIterationCount: defaultSettings.WarmupIterationCount, targetIterationCount: defaultSettings.TargetIterationCount));
}
return(attrs.Select(attr => attr.Task));
}
public void Throughput(BenchmarkSettings settings, long operationsPerInvoke, Action setupAction, Action targetAction, Action idleAction)
{
setupAction();
targetAction();
idleAction();
long invokeCount = 1;
double lastPreWarmupMilliseconds = 0;
while (true)
{
var measurement = MultiInvoke("// Pre-Warmup", setupAction, targetAction, invokeCount, operationsPerInvoke);
lastPreWarmupMilliseconds = measurement.Milliseconds;
if (lastPreWarmupMilliseconds > InvokeTimoutMilliseconds)
break;
if (lastPreWarmupMilliseconds < 1)
invokeCount *= InvokeTimoutMilliseconds;
else
invokeCount *= (long)Math.Ceiling(InvokeTimoutMilliseconds / lastPreWarmupMilliseconds);
}
double idleMilliseconds = 0;
for (int i = 0; i < Math.Min(3, settings.WarmupIterationCount); i++)
{
var measurement = MultiInvoke("// Warmup (idle)", setupAction, idleAction, invokeCount, operationsPerInvoke);
idleMilliseconds = measurement.Milliseconds;
}
invokeCount = invokeCount * 1000 / (long)Math.Round(Math.Min(1000, Math.Max(100, lastPreWarmupMilliseconds - idleMilliseconds)));
Console.WriteLine("// IterationCount = " + invokeCount);
long idleTicks = 0;
var targetIdleInvokeCount = Math.Min(5, settings.TargetIterationCount);
for (int i = 0; i < targetIdleInvokeCount; i++)
{
var measurement = MultiInvoke("// Target (idle)", setupAction, idleAction, invokeCount, operationsPerInvoke);
idleTicks += measurement.Ticks;
}
idleTicks /= targetIdleInvokeCount;
for (int i = 0; i < settings.WarmupIterationCount; i++)
MultiInvoke("// Warmup " + (i + 1), setupAction, targetAction, invokeCount, operationsPerInvoke, idleTicks);
for (int i = 0; i < settings.TargetIterationCount; i++)
MultiInvoke("Target " + (i + 1), setupAction, targetAction, invokeCount, operationsPerInvoke, idleTicks);
}
private static IEnumerable<Benchmark> CompetitionToBenchmarks(object competition, BenchmarkSettings defaultSettings)
{
if (defaultSettings == null)
defaultSettings = BenchmarkSettings.CreateDefault();
var targetType = competition.GetType();
var methods = targetType.GetMethods();
var setupMethod = methods.FirstOrDefault(m => m.ResolveAttribute<SetupAttribute>() != null);
if (setupMethod != null)
{
// setupMethod is optional, but if it's there it must have the correct signature, accessibility, etc
AssertMethodHasCorrectSignature("Setup", setupMethod);
AssertMethodIsAccessible("Setup", setupMethod);
AssertMethodIsNotDeclaredInGeneric("Setup", setupMethod);
AssertMethodIsNotGeneric("Setup", setupMethod);
}
for (int i = 0; i < methods.Length; i++)
{
var methodInfo = methods[i];
var benchmarkAttribute = methodInfo.ResolveAttribute<BenchmarkAttribute>();
if (benchmarkAttribute != null)
{
var target = new BenchmarkTarget(targetType, methodInfo, setupMethod, benchmarkAttribute.Description);
AssertMethodHasCorrectSignature("Benchmark", methodInfo);
AssertMethodIsAccessible("Benchmark", methodInfo);
AssertMethodIsNotDeclaredInGeneric("Benchmark", methodInfo);
AssertMethodIsNotGeneric("Benchmark", methodInfo);
foreach (var task in BenchmarkTask.Resolve(methodInfo, defaultSettings))
yield return new Benchmark(target, task);
}
}
}
public IEnumerable<BenchmarkReport> RunUrl(string url, BenchmarkSettings defaultSettings = null)
{
return RunCompetition(UrlToBenchmarks(url, defaultSettings).ToList());
}
public IEnumerable<BenchmarkReport> RunSource(string source, BenchmarkSettings defaultSettings = null)
{
return RunCompetition(SourceToBenchmarks(source, defaultSettings).ToList());
}
public IEnumerable<BenchmarkReport> RunCompetition(object benchmarkCompetition, BenchmarkSettings defaultSettings = null)
{
return RunCompetition(CompetitionToBenchmarks(benchmarkCompetition, defaultSettings).ToList());
}
private static IEnumerable<Benchmark> UrlToBenchmarks(string url, BenchmarkSettings defaultSettings)
{
string benchmarkContent = String.Empty;
try
{
var webRequest = WebRequest.Create(url);
using (var response = webRequest.GetResponse())
using (var content = response.GetResponseStream())
using (var reader = new StreamReader(content))
benchmarkContent = reader.ReadToEnd();
if (string.IsNullOrWhiteSpace(benchmarkContent))
{
Console.WriteLine($"content of '{url}' is empty.");
yield break;
}
}
catch (Exception e)
{
Console.WriteLine("Exception: " + e.Message);
yield break;
}
var cSharpCodeProvider = new CSharpCodeProvider();
var compilerParameters = new CompilerParameters(new[] { "mscorlib.dll", "System.Core.dll" }) { CompilerOptions = "/unsafe" };
compilerParameters.ReferencedAssemblies.Add(typeof(BenchmarkRunner).Assembly.Location);
var compilerResults = cSharpCodeProvider.CompileAssemblyFromSource(compilerParameters, benchmarkContent);
if (compilerResults.Errors.HasErrors)
{
compilerResults.Errors.Cast<CompilerError>().ToList().ForEach(error => Console.WriteLine(error.ErrorText));
yield break;
}
foreach (var type in compilerResults.CompiledAssembly.GetTypes())
{
var instance = Activator.CreateInstance(type);
foreach (var benchmark in CompetitionToBenchmarks(instance, defaultSettings))
yield return new Benchmark(new BenchmarkTarget(benchmark.Target.Type, benchmark.Target.Method, benchmark.Target.Description, benchmarkContent), benchmark.Task);
}
}
private static IEnumerable<Benchmark> CompetitionToBenchmarks(object competition, BenchmarkSettings defaultSettings)
{
if (defaultSettings == null)
defaultSettings = BenchmarkSettings.CreateDefault();
var targetType = competition.GetType();
var methods = targetType.GetMethods();
for (int i = 0; i < methods.Length; i++)
{
var methodInfo = methods[i];
var benchmarkAttribute = methodInfo.ResolveAttribute<BenchmarkAttribute>();
if (benchmarkAttribute != null)
{
var target = new BenchmarkTarget(targetType, methodInfo, benchmarkAttribute.Description);
AssertBenchmarkMethodHasCorrectSignature(methodInfo);
AssertBenchmarkMethodIsAccessible(methodInfo);
AssertBenchmarkMethodIsNotDeclaredInGeneric(methodInfo);
AssertBenchmarkMethodIsNotGeneric(methodInfo);
foreach (var task in BenchmarkTask.Resolve(methodInfo, defaultSettings))
yield return new Benchmark(target, task);
}
}
}
public BenchmarkTask(int processCount, BenchmarkConfiguration configuration, BenchmarkSettings settings)
{
ProcessCount = processCount;
Configuration = configuration;
Settings = settings;
}
public BenchmarkTask(int processCount, BenchmarkConfiguration configuration, BenchmarkSettings settings)
{
ProcessCount = processCount;
Configuration = configuration;
Settings = settings;
}
private static IEnumerable<Benchmark> CompetitionToBenchmarks(object competition, BenchmarkSettings defaultSettings)
{
if (defaultSettings == null)
defaultSettings = BenchmarkSettings.CreateDefault();
var targetType = competition.GetType();
var methods = targetType.GetMethods();
var setupMethod = methods.FirstOrDefault(m => m.ResolveAttribute<SetupAttribute>() != null);
if (setupMethod != null)
{
// setupMethod is optional, but if it's there it must have the correct signature, accessibility, etc
AssertMethodHasCorrectSignature("Setup", setupMethod);
AssertMethodIsAccessible("Setup", setupMethod);
AssertMethodIsNotDeclaredInGeneric("Setup", setupMethod);
AssertMethodIsNotGeneric("Setup", setupMethod);
}
// If there is one, get the single Field or Property that has the [Params(..)] attribute
var fields = targetType.GetFields().Select(f => new
{
f.Name,
Attribute = f.ResolveAttribute<ParamsAttribute>(),
IsStatic = f.IsStatic,
});
var properties = targetType.GetProperties().Select(f => new
{
f.Name,
Attribute = f.ResolveAttribute<ParamsAttribute>(),
IsStatic = f.GetSetMethod().IsStatic
});
var fieldOrProperty = fields.Concat(properties).FirstOrDefault(i => i.Attribute != null);
for (int i = 0; i < methods.Length; i++)
{
var methodInfo = methods[i];
var benchmarkAttribute = methodInfo.ResolveAttribute<BenchmarkAttribute>();
if (benchmarkAttribute != null)
{
var target = new BenchmarkTarget(targetType, methodInfo, setupMethod, benchmarkAttribute.Description);
AssertMethodHasCorrectSignature("Benchmark", methodInfo);
AssertMethodIsAccessible("Benchmark", methodInfo);
AssertMethodIsNotDeclaredInGeneric("Benchmark", methodInfo);
AssertMethodIsNotGeneric("Benchmark", methodInfo);
foreach (var task in BenchmarkTask.Resolve(methodInfo, defaultSettings))
{
if (fieldOrProperty == null)
{
yield return new Benchmark(target, task);
}
else
{
var @params = new BenchmarkParams(fieldOrProperty.Name, fieldOrProperty.IsStatic, fieldOrProperty.Attribute.Args);
// All the properties of BenchmarkTask and it's children are immutable, so cloning a BenchmarkTask like this should be safe
var newTask = new BenchmarkTask(task.ProcessCount, task.Configuration, task.Settings, @params);
yield return new Benchmark(target, newTask);
}
}
}
}
}
private static IEnumerable<Benchmark> SourceToBenchmarks(string source, BenchmarkSettings defaultSettings)
{
string benchmarkContent = source;
var cSharpCodeProvider = new CSharpCodeProvider();
var compilerParameters = new CompilerParameters(new[] { "mscorlib.dll", "System.Core.dll" }) { CompilerOptions = "/unsafe" };
compilerParameters.ReferencedAssemblies.Add(typeof(BenchmarkRunner).Assembly.Location);
var compilerResults = cSharpCodeProvider.CompileAssemblyFromSource(compilerParameters, benchmarkContent);
if (compilerResults.Errors.HasErrors)
{
compilerResults.Errors.Cast<CompilerError>().ToList().ForEach(error => Console.WriteLine(error.ErrorText));
yield break;
}
foreach (var type in compilerResults.CompiledAssembly.GetTypes())
{
var instance = Activator.CreateInstance(type);
foreach (var benchmark in CompetitionToBenchmarks(instance, defaultSettings))
{
yield return new Benchmark(new BenchmarkTarget(benchmark.Target.Type,
benchmark.Target.Method,
benchmark.Target.SetupMethod,
benchmark.Target.Description,
benchmarkContent),
benchmark.Task);
}
}
}
private static IEnumerable<Benchmark> UrlToBenchmarks(string url, BenchmarkSettings defaultSettings)
{
string benchmarkContent = String.Empty;
try
{
var webRequest = WebRequest.Create(url);
using (var response = webRequest.GetResponse())
using (var content = response.GetResponseStream())
using (var reader = new StreamReader(content))
benchmarkContent = reader.ReadToEnd();
if (string.IsNullOrWhiteSpace(benchmarkContent))
{
Console.WriteLine($"content of '{url}' is empty.");
return new Benchmark[0];
}
}
catch (Exception e)
{
Console.WriteLine("Exception: " + e.Message);
return new Benchmark[0];
}
return SourceToBenchmarks(benchmarkContent, defaultSettings);
}
public void RunBenchmark(BenchmarkSettings settings)
{
new BenchmarkMethodInvoker().Throughput(settings, 1, setupAction, targetAction, idleAction);
}
