private static IEnumerable <Benchmark> CompetitionToBenchmarks(object competition, BenchmarkSettings defaultSettings)
{
if (defaultSettings == null)
{
defaultSettings = BenchmarkSettings.CreateDefault();
}
var targetType = competition.GetType();
var methods = targetType.GetMethods();
for (int i = 0; i < methods.Length; i++)
{
var methodInfo = methods[i];
var benchmarkAttribute = methodInfo.ResolveAttribute <BenchmarkAttribute>();
if (benchmarkAttribute != null)
{
var target = new BenchmarkTarget(targetType, methodInfo, benchmarkAttribute.Description);
AssertBenchmarkMethodHasCorrectSignature(methodInfo);
AssertBenchmarkMethodIsAccessible(methodInfo);
AssertBenchmarkMethodIsNotDeclaredInGeneric(methodInfo);
AssertBenchmarkMethodIsNotGeneric(methodInfo);
foreach (var task in BenchmarkTask.Resolve(methodInfo, defaultSettings))
{
yield return(new Benchmark(target, task));
}
}
}
}
public static IEnumerable <Benchmark> TypeToBenchmarks(Type type)
{
var methods = type.GetMethods();
var setupMethod = methods.FirstOrDefault(m => m.ResolveAttribute <SetupAttribute>() != null);
if (setupMethod != null)
{
// setupMethod is optional, but if it's there it must have the correct signature, accessibility, etc
AssertMethodHasCorrectSignature("Setup", setupMethod);
AssertMethodIsAccessible("Setup", setupMethod);
AssertMethodIsNotGeneric("Setup", setupMethod);
}
// If there is one, get the single Field or Property that has the [Params(..)] attribute
var fields = type.GetFields().Select(f => new
{
f.Name,
Attribute = f.ResolveAttribute <ParamsAttribute>(),
IsStatic = f.IsStatic,
});
var properties = type.GetProperties().Select(f => new
{
f.Name,
Attribute = f.ResolveAttribute <ParamsAttribute>(),
IsStatic = f.GetSetMethod().IsStatic
});
var fieldOrProperty = fields.Concat(properties).FirstOrDefault(i => i.Attribute != null);
for (int i = 0; i < methods.Length; i++)
{
var methodInfo = methods[i];
var benchmarkAttribute = methodInfo.ResolveAttribute <BenchmarkAttribute>();
if (benchmarkAttribute != null)
{
var target = new BenchmarkTarget(type, methodInfo, setupMethod, benchmarkAttribute.Description);
AssertMethodHasCorrectSignature("Benchmark", methodInfo);
AssertMethodIsAccessible("Benchmark", methodInfo);
AssertMethodIsNotGeneric("Benchmark", methodInfo);
foreach (var task in BenchmarkTask.Resolve(methodInfo))
{
if (fieldOrProperty == null)
{
yield return(new Benchmark(target, task));
}
else
{
var parametersSets = new BenchmarkParametersSets(fieldOrProperty.Name, fieldOrProperty.IsStatic, fieldOrProperty.Attribute.Args);
// All the properties of BenchmarkTask and it's children are immutable, so cloning a BenchmarkTask like this should be safe
var newTask = new BenchmarkTask(task.ProcessCount, task.Configuration, parametersSets);
yield return(new Benchmark(target, newTask));
}
}
}
}
}
/** <inheritdoc/> */
public void Initialize(BenchmarkBase benchmark, ICollection <string> opNames)
{
_benchmarks = new Dictionary <string, BenchmarkTask>(opNames.Count);
// 1. Create folder for results.
var now = DateTime.Now;
var suffix = "-t=" + benchmark.Threads + "-d=" + benchmark.Duration +
"-w=" + benchmark.Warmup;
var path = benchmark.ResultFolder + "\\" + now.ToString("yyyyMMdd-HHmmss", Culture) + "-" +
benchmark.GetType().Name + suffix;
if (Directory.Exists(path))
{
Directory.Delete(path, true);
}
Directory.CreateDirectory(path);
var dateStr = "--Created " + DateTime.Now.ToString("yyyyMMdd-HHmmss", Culture);
var cfgStr = "--Benchmark config: " + benchmark;
// 2. For each operation create separate folder and initialize probe files there.
foreach (var opName in opNames)
{
var opDesc = benchmark.GetType().Name + "-" + opName + suffix;
var opPath = path + "\\" + opDesc;
Directory.CreateDirectory(opPath);
var task = new BenchmarkTask(opPath + "\\" + ProbePercentile,
opPath + "\\" + ProbeThroughput);
_benchmarks[opName] = task;
File.AppendAllText(task.FilePercentile, dateStr + "\n");
File.AppendAllText(task.FilePercentile, cfgStr + "\n");
File.AppendAllText(task.FilePercentile, "--Description: " + opDesc + "\n");
File.AppendAllText(task.FilePercentile, "@@" + benchmark.GetType().Name + "\n");
File.AppendAllText(task.FilePercentile, HdrPercentile + "\n");
File.AppendAllText(task.FileThroughput, dateStr + "\n");
File.AppendAllText(task.FileThroughput, cfgStr + "\n");
File.AppendAllText(task.FileThroughput, "--Description: " + opDesc + "\n");
File.AppendAllText(task.FileThroughput, "@@" + benchmark.GetType().Name + "\n");
File.AppendAllText(task.FileThroughput, HdrThroughput + "\n");
}
// 3. Start writer thread.
new Thread(_writer.Run).Start();
}
/** <inheritdoc/> */
public void Initialize(BenchmarkBase benchmark, ICollection<string> opNames)
{
_benchmarks = new Dictionary<string, BenchmarkTask>(opNames.Count);
// 1. Create folder for results.
var now = DateTime.Now;
var suffix = "-t=" + benchmark.Threads + "-d=" + benchmark.Duration +
"-w=" + benchmark.Warmup;
var path = benchmark.ResultFolder + "\\" + now.ToString("yyyyMMdd-HHmmss", Culture) + "-" +
benchmark.GetType().Name + suffix;
if (Directory.Exists(path))
Directory.Delete(path, true);
Directory.CreateDirectory(path);
var dateStr = "--Created " + DateTime.Now.ToString("yyyyMMdd-HHmmss", Culture);
var cfgStr = "--Benchmark config: " + benchmark;
// 2. For each operation create separate folder and initialize probe files there.
foreach (var opName in opNames)
{
var opDesc = benchmark.GetType().Name + "-" + opName + suffix;
var opPath = path + "\\" + opDesc;
Directory.CreateDirectory(opPath);
var task = new BenchmarkTask(opPath + "\\" + ProbePercentile,
opPath + "\\" + ProbeThroughput);
_benchmarks[opName] = task;
File.AppendAllText(task.FilePercentile, dateStr + "\n");
File.AppendAllText(task.FilePercentile, cfgStr + "\n");
File.AppendAllText(task.FilePercentile, "--Description: " + opDesc + "\n");
File.AppendAllText(task.FilePercentile, "@@" + benchmark.GetType().Name + "\n");
File.AppendAllText(task.FilePercentile, HdrPercentile + "\n");
File.AppendAllText(task.FileThroughput, dateStr + "\n");
File.AppendAllText(task.FileThroughput, cfgStr + "\n");
File.AppendAllText(task.FileThroughput, "--Description: " + opDesc + "\n");
File.AppendAllText(task.FileThroughput, "@@" + benchmark.GetType().Name + "\n");
File.AppendAllText(task.FileThroughput, HdrThroughput + "\n");
}
// 3. Start writer thread.
new Thread(_writer.Run).Start();
}
public void SingleRun(BenchmarkTask task, long operationsPerInvoke, Action setupAction, Action targetAction, Action idleAction)
{
for (int i = 0; i < task.Configuration.WarmupIterationCount; i++)
{
BenchmarkState.Instance.IterationMode = BenchmarkIterationMode.Warmup;
BenchmarkState.Instance.Iteration = i;
MultiInvoke("// Warmup " + (i + 1), setupAction, targetAction, 1, operationsPerInvoke);
}
for (int i = 0; i < task.Configuration.TargetIterationCount; i++)
{
BenchmarkState.Instance.IterationMode = BenchmarkIterationMode.Target;
BenchmarkState.Instance.Iteration = i;
MultiInvoke("Target " + (i + 1), setupAction, targetAction, 1, operationsPerInvoke);
}
}
/// <summary>
/// Run the benchmark.
/// </summary>
public void Run()
{
PrintDebug("Started benchmark: " + this);
ValidateArguments();
if (ResultWriter.ToLower().Equals(ResultWriterConsole))
{
_writer = new BenchmarkConsoleResultWriter();
}
else
{
_writer = new BenchmarkFileResultWriter();
}
OnStarted();
PrintDebug("Benchmark setup finished.");
try
{
_descs = new List <BenchmarkOperationDescriptor>();
GetDescriptors(_descs);
if (_descs.Count == 0)
{
throw new Exception("No tasks provided for benchmark.");
}
// Initialize writer.
var opNames = new List <string>(_descs.Select(desc => desc.Name));
PrintDebug(() =>
{
var sb = new StringBuilder("Operations: ");
foreach (var opName in opNames)
{
sb.Append(opName).Append(" ");
}
return(sb.ToString());
});
_writer.Initialize(this, opNames);
PrintDebug("Initialized result writer.");
// Start worker threads.
_tasks = new List <BenchmarkTask>(Threads);
PrintDebug("Starting worker threads: " + Threads);
for (var i = 0; i < Threads; i++)
{
var task = new BenchmarkTask(this, _descs);
_tasks.Add(task);
new Thread(task.Run).Start();
}
PrintDebug("Waiting worker threads to start: " + Threads);
// Await for all threads to start in spin loop.
while (Thread.VolatileRead(ref _readyThreads) < Threads)
{
Thread.Sleep(10);
}
PrintDebug("Worker threads started: " + Threads);
// Start throughput writer thread.
var writerThread = new Thread(new ThroughputTask(this).Run)
{
IsBackground = true
};
writerThread.Start();
PrintDebug("Started throughput writer thread.");
// Start warmup thread if needed.
if (Warmup > 0)
{
var thread = new Thread(new WarmupTask(this, Warmup).Run)
{
IsBackground = true
};
thread.Start();
PrintDebug("Started warmup timeout thread: " + Warmup);
}
else
{
_warmup = false;
}
_barrier = new Barrier(Threads, b =>
{
Console.WriteLine("Warmup finished.");
_totalWatch.Start();
});
// Start timeout thread if needed.
if (Duration > 0)
{
if (Operations > 0)
{
PrintDebug("Duration argument is ignored because operations number is set: " +
Operations);
}
else
{
var thread = new Thread(new TimeoutTask(this, Warmup + Duration).Run)
{
IsBackground = true
};
thread.Start();
PrintDebug("Started duration timeout thread: " + Duration);
}
}
// Let workers start execution.
_start = true;
// Await workers completion.
PrintDebug("Awaiting worker threads completion.");
Monitor.Enter(this);
try
{
while (_finishedThreads < Threads)
{
Monitor.Wait(this);
}
}
finally
{
Monitor.Exit(this);
}
PrintDebug("Worker threads completed.");
}
finally
{
OnFinished();
_totalWatch.Stop();
PrintDebug("Tear down invoked.");
if (PrintThroughputInfo())
{
var avgThroughput = _totalWatch.ElapsedMilliseconds == 0
? 0
: _curOps * 1000 / _totalWatch.ElapsedMilliseconds;
var avgLatency = _curOps == 0
? 0
: (double)_totalWatch.ElapsedMilliseconds * Threads / _curOps;
Console.WriteLine("Finishing benchmark [name=" + GetType().Name +
", time=" + _totalWatch.ElapsedMilliseconds +
"ms, ops=" + _curOps +
", threads=" + Threads +
", avgThroughput=" + avgThroughput +
", avgLatency=" + string.Format("{0:0.000}ms", avgLatency) + ']');
}
else
{
Console.WriteLine("Finishing benchmark [name=" + GetType().Name +
", time=" + _totalWatch.ElapsedMilliseconds +
"ms, ops=" + Operations +
", threads=" + Threads + ']');
}
}
_percentiles = new Dictionary <string, long[]>(_descs.Count);
foreach (var desc in _descs)
{
_percentiles[desc.Name] = new long[ResultBucketCount];
}
foreach (var task in _tasks)
{
task.CollectPercentiles(_percentiles);
}
foreach (var percentile in _percentiles)
{
_writer.WritePercentiles(percentile.Key, ResultBucketInterval, percentile.Value);
}
_writer.Commit();
PrintDebug("Results committed to output writer.");
}
public Benchmark(BenchmarkTarget target, BenchmarkTask task)
{
Target = target;
Task = task;
}
/// <summary>
/// Run the benchmark.
/// </summary>
public void Run()
{
PrintDebug("Started benchmark: " + this);
ValidateArguments();
if (ResultWriter.ToLower().Equals(ResultWriterConsole))
_writer = new BenchmarkConsoleResultWriter();
else
_writer = new BenchmarkFileResultWriter();
OnStarted();
PrintDebug("Benchmark setup finished.");
try
{
_descs = new List<BenchmarkOperationDescriptor>();
GetDescriptors(_descs);
if (_descs.Count == 0)
throw new Exception("No tasks provided for benchmark.");
// Initialize writer.
var opNames = new List<string>(_descs.Select(desc => desc.Name));
PrintDebug(() =>
{
var sb = new StringBuilder("Operations: ");
foreach (var opName in opNames)
sb.Append(opName).Append(" ");
return sb.ToString();
});
_writer.Initialize(this, opNames);
PrintDebug("Initialized result writer.");
// Start worker threads.
_tasks = new List<BenchmarkTask>(Threads);
PrintDebug("Starting worker threads: " + Threads);
for (var i = 0; i < Threads; i++)
{
var task = new BenchmarkTask(this, _descs);
_tasks.Add(task);
new Thread(task.Run).Start();
}
PrintDebug("Waiting worker threads to start: " + Threads);
// Await for all threads to start in spin loop.
while (Thread.VolatileRead(ref _readyThreads) < Threads)
Thread.Sleep(10);
PrintDebug("Worker threads started: " + Threads);
// Start throughput writer thread.
var writerThread = new Thread(new ThroughputTask(this).Run) {IsBackground = true};
writerThread.Start();
PrintDebug("Started throughput writer thread.");
// Start warmup thread if needed.
if (Warmup > 0)
{
var thread = new Thread(new WarmupTask(this, Warmup).Run) {IsBackground = true};
thread.Start();
PrintDebug("Started warmup timeout thread: " + Warmup);
}
else
_warmup = false;
_barrier = new Barrier(Threads, b =>
{
Console.WriteLine("Warmup finished.");
_totalWatch.Start();
});
// Start timeout thread if needed.
if (Duration > 0)
{
if (Operations > 0)
PrintDebug("Duration argument is ignored because operations number is set: " +
Operations);
else
{
var thread = new Thread(new TimeoutTask(this, Warmup + Duration).Run) {IsBackground = true};
thread.Start();
PrintDebug("Started duration timeout thread: " + Duration);
}
}
// Let workers start execution.
_start = true;
// Await workers completion.
PrintDebug("Awaiting worker threads completion.");
Monitor.Enter(this);
try
{
while (_finishedThreads < Threads)
Monitor.Wait(this);
}
finally
{
Monitor.Exit(this);
}
PrintDebug("Worker threads completed.");
}
finally
{
OnFinished();
_totalWatch.Stop();
PrintDebug("Tear down invoked.");
if (PrintThroughputInfo())
{
var avgThroughput = _totalWatch.ElapsedMilliseconds == 0
? 0
: _curOps*1000/_totalWatch.ElapsedMilliseconds;
var avgLatency = _curOps == 0
? 0
: (double) _totalWatch.ElapsedMilliseconds*Threads/_curOps;
Console.WriteLine("Finishing benchmark [name=" + GetType().Name +
", time=" + _totalWatch.ElapsedMilliseconds +
"ms, ops=" + _curOps +
", threads=" + Threads +
", avgThroughput=" + avgThroughput +
", avgLatency=" + string.Format("{0:0.000}ms", avgLatency) + ']');
}
else
{
Console.WriteLine("Finishing benchmark [name=" + GetType().Name +
", time=" + _totalWatch.ElapsedMilliseconds +
"ms, ops=" + Operations +
", threads=" + Threads + ']');
}
}
_percentiles = new Dictionary<string, long[]>(_descs.Count);
foreach (var desc in _descs)
_percentiles[desc.Name] = new long[ResultBucketCount];
foreach (var task in _tasks)
task.CollectPercentiles(_percentiles);
foreach (var percentile in _percentiles)
_writer.WritePercentiles(percentile.Key, ResultBucketInterval, percentile.Value);
_writer.Commit();
PrintDebug("Results committed to output writer.");
}
public void Throughput(BenchmarkTask task, long operationsPerInvoke, Action setupAction, Action targetAction, Action idleAction)
{
setupAction();
targetAction();
idleAction();
long invokeCount = 1;
double lastPreWarmupMilliseconds = 0;
int preWarmupCounter = 0;
while (true)
{
BenchmarkState.Instance.IterationMode = BenchmarkIterationMode.PreWarmup;
BenchmarkState.Instance.Iteration = preWarmupCounter++;
var measurement = MultiInvoke("// Pre-Warmup", setupAction, targetAction, invokeCount, operationsPerInvoke);
lastPreWarmupMilliseconds = measurement.Milliseconds;
if (lastPreWarmupMilliseconds > InvokeTimoutMilliseconds)
{
break;
}
if (lastPreWarmupMilliseconds < 1)
{
invokeCount *= InvokeTimoutMilliseconds;
}
else
{
invokeCount *= (long)Math.Ceiling(InvokeTimoutMilliseconds / lastPreWarmupMilliseconds);
}
}
double idleMilliseconds = 0;
for (int i = 0; i < Math.Min(3, task.Configuration.WarmupIterationCount); i++)
{
BenchmarkState.Instance.IterationMode = BenchmarkIterationMode.WarmupIdle;
BenchmarkState.Instance.Iteration = i;
var measurement = MultiInvoke("// Warmup (idle)", setupAction, idleAction, invokeCount, operationsPerInvoke);
idleMilliseconds = measurement.Milliseconds;
}
invokeCount = invokeCount * 1000 / (long)Math.Round(Math.Min(1000, Math.Max(100, lastPreWarmupMilliseconds - idleMilliseconds)));
Console.WriteLine("// IterationCount = " + invokeCount);
long idleTicks = 0;
var targetIdleInvokeCount = Math.Min(5, task.Configuration.TargetIterationCount);
for (int i = 0; i < targetIdleInvokeCount; i++)
{
BenchmarkState.Instance.IterationMode = BenchmarkIterationMode.TargetIdle;
BenchmarkState.Instance.Iteration = i;
var measurement = MultiInvoke("// Target (idle)", setupAction, idleAction, invokeCount, operationsPerInvoke);
idleTicks += measurement.Ticks;
}
idleTicks /= targetIdleInvokeCount;
for (int i = 0; i < task.Configuration.WarmupIterationCount; i++)
{
BenchmarkState.Instance.IterationMode = BenchmarkIterationMode.Warmup;
BenchmarkState.Instance.Iteration = i;
MultiInvoke("// Warmup " + (i + 1), setupAction, targetAction, invokeCount, operationsPerInvoke, idleTicks);
}
for (int i = 0; i < task.Configuration.TargetIterationCount; i++)
{
BenchmarkState.Instance.IterationMode = BenchmarkIterationMode.Target;
BenchmarkState.Instance.Iteration = i;
MultiInvoke("Target " + (i + 1), setupAction, targetAction, invokeCount, operationsPerInvoke, idleTicks);
}
}
private void runTaskButton_Click(object sender, RoutedEventArgs e)
{
Tasks.Task task;
string taskName = tasksBox.Text;
switch (taskName)
{
case "Benchmark":
try
{
task = new BenchmarkTask(labelBox.Text, productBox.Text, parametersBox.Text, int.Parse(loopsBox.Text), ComputerWindow.SelectedComputers(), parseOSList);
}
catch (FormatException)
{
MessageBox.Show("ERROR", "Loops is not a number!");
return;
}
break;
case "Boot":
task = new BootTask(ComputerWindow.SelectedComputers(), parseOSList);
break;
case "Reboot":
task = new RebootTask(ComputerWindow.SelectedComputers(), parseOSList);
break;
case "Shutdown":
task = new ShutdownTask(ComputerWindow.SelectedComputers(), parseOSList);
break;
case "Exec":
task = new ExecTask(commandTextBox.Text, ComputerWindow.SelectedComputers(), parseOSList);
break;
case "Winupdate":
task = new WinUpdateTask(ComputerWindow.SelectedComputers(), parseOSList);
break;
case "Update":
task = new UpdateTask(ComputerWindow.SelectedComputers(), parseOSList);
break;
case "Valeta/Dandia":
try
{
task = new BatTask(zipBox.Text, outputPathcBox.Text, int.Parse(countBox.Text), modeBox.Text, (BatTask.BatType)batTypeBox.SelectedIndex, nocopyBox.IsChecked.Value, ComputerWindow.SelectedComputers(), parseOSList);
}
catch (FormatException)
{
MessageBox.Show("ERROR", "Test count is not a number!");
return;
}
break;
default:
return;
}
List <Tuple <string, object> > parameters = task.GetJackParameters();
PowerShell psinstance = PowerShell.Create();
psinstance.Streams.Verbose.DataAdded += jackInformationEventHandler;
psinstance.Streams.Progress.DataAdded += jackProgressEventHandler;
psinstance.Streams.Debug.DataAdded += jackDebugEventHandler;
psinstance.Streams.Error.DataAdded += jackErrorEventHandler;
//scriptInvoker.Invoke("Set-ExecutionPolicy Unrestricted Process");
psinstance.AddCommand("Set-ExecutionPolicy");
psinstance.AddParameter("-ExecutionPolicy", "Bypass");
psinstance.AddCommand(SCRIPT_PATH);
foreach (var param in parameters)
{
psinstance.AddParameter(param.Item1, param.Item2);
}
var results = psinstance.BeginInvoke();
/*foreach (PSObject result in results)
* {
* AppendOutputText(result.ToString());
* Console.WriteLine(result.ToString());
* }*/
ListBoxItem item = new ListBoxItem();
item.Content = task.ToString();
taskHistoryBox.Items.Add(task);
}
