public BenchmarkConfiguration(BenchmarkMode mode, BenchmarkPlatform platform, BenchmarkJitVersion jitVersion, BenchmarkFramework framework)
{
Mode = mode;
Platform = platform;
JitVersion = jitVersion;
Framework = framework;
}
public BenchmarkConfiguration(BenchmarkMode mode, BenchmarkPlatform platform, BenchmarkJitVersion jitVersion, BenchmarkFramework framework)
{
Mode = mode;
Platform = platform;
JitVersion = jitVersion;
Framework = framework;
}
public TaskAttribute(
int processCount = 1,
BenchmarkMode mode = BenchmarkMode.Throughput,
BenchmarkPlatform platform = BenchmarkPlatform.HostPlatform,
BenchmarkJitVersion jitVersion = BenchmarkJitVersion.HostJit,
BenchmarkFramework framework = BenchmarkFramework.HostFramework,
int warmupIterationCount = 5,
int targetIterationCount = 10
)
{
Task = new BenchmarkTask(
processCount,
new BenchmarkConfiguration(mode, platform, jitVersion, framework, BenchmarkToolchain.Classic, BenchmarkRuntime.Clr, warmupIterationCount, targetIterationCount));
}
public TaskAttribute(
int processCount = 1,
BenchmarkMode mode = BenchmarkMode.Throughput,
BenchmarkPlatform platform = BenchmarkPlatform.HostPlatform,
BenchmarkJitVersion jitVersion = BenchmarkJitVersion.HostJit,
BenchmarkFramework framework = BenchmarkFramework.HostFramework,
int warmupIterationCount = 5,
int targetIterationCount = 10
)
{
Task = new BenchmarkTask(
processCount,
new BenchmarkConfiguration(mode, platform, jitVersion, framework, BenchmarkExecutor.Classic, BenchmarkRuntime.Clr, string.Empty, warmupIterationCount, targetIterationCount));
}
public TaskAttribute(
int processCount = 1,
BenchmarkMode mode = BenchmarkMode.Throughput,
BenchmarkPlatform platform = BenchmarkPlatform.HostPlatform,
BenchmarkJitVersion jitVersion = BenchmarkJitVersion.HostJit,
BenchmarkFramework framework = BenchmarkFramework.HostFramework,
int warmupIterationCount = 5,
int targetIterationCount = 10
)
{
Task = new BenchmarkTask(
processCount,
new BenchmarkConfiguration(mode, platform, jitVersion, framework),
new BenchmarkSettings(warmupIterationCount, targetIterationCount));
}
public BenchmarkTaskAttribute(
int processCount = 3,
BenchmarkMode mode = BenchmarkMode.Throughput,
BenchmarkPlatform platform = BenchmarkPlatform.HostPlatform,
BenchmarkJitVersion jitVersion = BenchmarkJitVersion.HostJit,
BenchmarkFramework framework = BenchmarkFramework.HostFramework,
int warmupIterationCount = 5,
int targetIterationCount = 10
)
{
Task = new BenchmarkTask(
processCount,
new BenchmarkConfiguration(mode, platform, jitVersion, framework),
new BenchmarkSettings(warmupIterationCount, targetIterationCount));
}
public BenchmarkTaskAttribute(
int processCount = 3,
BenchmarkMode mode = BenchmarkMode.Throughput,
BenchmarkPlatform platform = BenchmarkPlatform.HostPlatform,
BenchmarkJitVersion jitVersion = BenchmarkJitVersion.HostJit,
BenchmarkFramework framework = BenchmarkFramework.HostFramework,
BenchmarkToolchain toolchain = BenchmarkToolchain.Classic,
BenchmarkRuntime runtime = BenchmarkRuntime.Clr,
int warmupIterationCount = 5,
int targetIterationCount = 10
)
{
Task = new BenchmarkTask(
processCount,
new BenchmarkConfiguration(mode, platform, jitVersion, framework, toolchain, runtime, warmupIterationCount, targetIterationCount));
}
public BenchmarkTaskAttribute(
int processCount = 3,
BenchmarkMode mode = BenchmarkMode.Throughput,
BenchmarkPlatform platform = BenchmarkPlatform.HostPlatform,
BenchmarkJitVersion jitVersion = BenchmarkJitVersion.HostJit,
BenchmarkFramework framework = BenchmarkFramework.HostFramework,
BenchmarkExecutor executor = BenchmarkExecutor.Classic,
BenchmarkRuntime runtime = BenchmarkRuntime.Clr,
string runtimeVersion = null,
int warmupIterationCount = 5,
int targetIterationCount = 10,
int[] intParams = null
)
{
Task = new BenchmarkTask(
processCount,
new BenchmarkConfiguration(mode, platform, jitVersion, framework, executor, runtime, runtimeVersion, warmupIterationCount, targetIterationCount),
new BenchmarkParametersSets(intParams));
}
public BenchmarkConfiguration(
BenchmarkMode mode,
BenchmarkPlatform platform,
BenchmarkJitVersion jitVersion,
BenchmarkFramework framework,
BenchmarkToolchain toolchain,
BenchmarkRuntime runtime,
int warmupIterationCount,
int targetIterationCount)
{
Mode = mode;
Platform = platform;
JitVersion = jitVersion;
Framework = framework;
Toolchain = toolchain;
Runtime = runtime;
WarmupIterationCount = warmupIterationCount;
TargetIterationCount = targetIterationCount;
}
public BenchmarkConfiguration(
BenchmarkMode mode,
BenchmarkPlatform platform,
BenchmarkJitVersion jitVersion,
BenchmarkFramework framework,
BenchmarkToolchain toolchain,
BenchmarkRuntime runtime,
int warmupIterationCount,
int targetIterationCount)
{
Mode = mode;
Platform = platform;
JitVersion = jitVersion;
Framework = framework;
Toolchain = toolchain;
Runtime = runtime;
WarmupIterationCount = warmupIterationCount;
TargetIterationCount = targetIterationCount;
}
private static void RunPredefinedBenchmarks(BenchmarkMode mode)
{
if ((mode & BenchmarkMode.Performance) != 0)
{
RunBenchmark <UnreliablePerformanceBenchmark>();
RunBenchmark <ReliablePerformanceBenchmark>();
Console.WriteLine($"Finished {BenchmarkMode.Performance} Benchmark");
}
if ((mode & BenchmarkMode.Quick) != 0)
{
RunBenchmark <QuickBenchmark>();
Console.WriteLine($"Finished {BenchmarkMode.Quick} Benchmark");
}
if ((mode & BenchmarkMode.Sampling) != 0)
{
RunBenchmark <SamplingBenchmark>();
Console.WriteLine($"Finished {BenchmarkMode.Sampling} Benchmark");
}
}
public BenchmarkConfiguration(
BenchmarkMode mode,
BenchmarkPlatform platform,
BenchmarkJitVersion jitVersion,
BenchmarkFramework framework,
BenchmarkExecutor executor,
BenchmarkRuntime runtime,
string runtimeVersion,
int warmupIterationCount,
int targetIterationCount)
{
Mode = mode;
Platform = platform;
JitVersion = jitVersion;
Framework = framework;
Executor = executor;
Runtime = runtime;
RuntimeVersion = runtimeVersion;
WarmupIterationCount = warmupIterationCount;
TargetIterationCount = targetIterationCount;
}
static void Main(string[] args)
{
if (args.Length == 0)
{
PrintUsage();
return;
}
BenchmarkMode mode = BenchmarkMode.Normal;
String basearguments = null, arguments = null;
String filename = null;
bool firstVerbose = false;
int startSize = 100000;
int endSize = 1000000;
int stepSize = 100000;
for (int i = 0; i < args.Length; i++)
{
if (args[i][0] == '-')
{
if (args[i] == "-grshellmutex")
{
if (mode != BenchmarkMode.Normal)
{
Console.Error.WriteLine("Another mode has already been specified!");
PrintUsage();
return;
}
if (i + 1 >= args.Length)
{
Console.Error.WriteLine("Test application not specified!");
PrintUsage();
return;
}
mode = BenchmarkMode.GrShellMutex;
}
else if (args[i] == "-grshell")
{
if (mode != BenchmarkMode.Normal)
{
Console.Error.WriteLine("Another mode has already been specified!");
PrintUsage();
return;
}
if (i + 1 >= args.Length)
{
Console.Error.WriteLine("Test application not specified!");
PrintUsage();
return;
}
mode = BenchmarkMode.GrShell;
}
else if (args[i] == "-nosize")
{
if (mode != BenchmarkMode.Normal)
{
Console.Error.WriteLine("Another mode has already been specified!");
PrintUsage();
return;
}
mode = BenchmarkMode.NoSize;
startSize = 0;
endSize = 0;
stepSize = 1;
}
else if (args[i] == "-start")
{
if (i + 1 >= args.Length)
{
Console.Error.WriteLine("Start size not specified!");
PrintUsage();
return;
}
startSize = Int32.Parse(args[++i]);
}
else if (args[i] == "-end")
{
if (i + 1 >= args.Length)
{
Console.Error.WriteLine("End size not specified!");
PrintUsage();
return;
}
endSize = Int32.Parse(args[++i]);
}
else if (args[i] == "-step")
{
if (i + 1 >= args.Length)
{
Console.Error.WriteLine("Step size not specified!");
PrintUsage();
return;
}
stepSize = Int32.Parse(args[++i]);
}
else if (args[i] == "-firstverbose")
{
firstVerbose = true;
}
else
{
Console.Error.WriteLine("Illegal option: \"" + args[i] + "\"");
PrintUsage();
return;
}
}
else
{
filename = args[i];
basearguments = String.Join(" ", args, i + 1, args.Length - i - 1) + " ";
break;
}
}
Console.Error.WriteLine("\"" + filename + "\": " + mode + " (" + startSize
+ " - " + endSize + ") += " + stepSize);
int BIGN = 4;
int N = 10;
// int BIGN = 2;
// int N = 2;
Console.WriteLine("{0,10};{1,10};{2,10};{3,10}", "size", "time", "benchtime", "memory");
for (int bigi = 0; bigi < BIGN; bigi++)
{
Console.Error.WriteLine(bigi + "");
for (int SIZE = startSize; SIZE <= endSize; SIZE += stepSize)
{
if (mode == BenchmarkMode.GrShellMutex)
{
arguments = basearguments + "-C \"select backend '../bin/lgspBackend.dll';;"
// + "new graph '..\\lib\\lgsp-MutexModel.dll' 'mutex graph';;"
// + "select actions '..\\lib\\lgsp-MutexActions.dll';;"
+ "new graph '../lib/lgsp-MutexPimpedModel.dll' 'mutex graph';;"
+ "select actions '../lib/lgsp-MutexPimpedActions.dll';;"
+ "new p1:Process;;"
+ "new p2:Process;;"
+ "new p1-:next->p2;;"
+ "new p2-:next->p1;;"
// + "grs newRule{" + (SIZE - 2) + "} ; mountRule ; requestRule{" + SIZE + "};;"
// + "custom graph analyze;;"
// + "custom actions gen_searchplan takeRule releaseRule giveRule;;"
// + "grs (takeRule ; releaseRule ; giveRule){" + SIZE + "}\";;"
+ "grs newRule{" + (SIZE - 2) + "} ; mountRule ; requestRule{" + SIZE + "} ;"
+ "(takeRule ; releaseRule ; giveRule){" + SIZE + "};;"
+ "quit\"";
}
else if (mode == BenchmarkMode.GrShell)
{
arguments = basearguments;
}
else
{
arguments = basearguments + SIZE;
}
if (firstVerbose)
{
Console.WriteLine("basearguments = >>>" + basearguments + "<<<");
Console.WriteLine("arguments = >>>" + arguments + "<<<");
}
Process benchTarget = new Process();
benchTarget.StartInfo.UseShellExecute = false;
benchTarget.StartInfo.RedirectStandardOutput = !firstVerbose;
benchTarget.StartInfo.FileName = filename;
benchTarget.StartInfo.Arguments = arguments;
benchTarget.Start();
long peakPagedMem = 0;
do
{
try
{
benchTarget.Refresh();
peakPagedMem = benchTarget.PeakPagedMemorySize64; // TODO: Not supported by Mono yet...
}
catch (Exception ex)
{
Console.WriteLine("Exception occurred while accessing PeakVirtualMemorySize64: " + ex.Message);
}
}while(!benchTarget.WaitForExit(50));
try
{
for (int i = 0; i < N; i++)
{
int startms = Environment.TickCount;
benchTarget = new Process();
benchTarget.StartInfo.UseShellExecute = false;
benchTarget.StartInfo.RedirectStandardOutput = true;
benchTarget.StartInfo.FileName = filename;
benchTarget.StartInfo.Arguments = arguments;
benchTarget.Start();
StreamReader reader = benchTarget.StandardOutput;
int time = 0;
String str;
while ((str = reader.ReadLine()) != null)
{
if (mode == BenchmarkMode.GrShell || mode == BenchmarkMode.GrShellMutex)
{
if (str.Length > 0 && (str[str.Length - 1] == ':' || str[str.Length - 1] == '.') && str[str.Length - 2] == 's' &&
str[str.Length - 3] == 'm' && str[str.Length - 4] == ' ')
{
int timeStartIndex = str.LastIndexOf(' ', str.Length - 5);
String timeStr = str.Substring(timeStartIndex + 1, str.Length - 5 - timeStartIndex);
int curTime = Int32.Parse(timeStr);
time += curTime;
}
}
else if (str.EndsWith(" ms"))
{
int timeStartIndex = str.LastIndexOf(' ', str.Length - 4);
String timeStr = str.Substring(timeStartIndex + 1, str.Length - 4 - timeStartIndex);
int curTime = Int32.Parse(timeStr);
time += curTime;
}
}
benchTarget.Close();
int benchtime = Environment.TickCount - startms;
Console.WriteLine("{0,10};{1,10};{2,10};{3,10}", SIZE, time, benchtime, peakPagedMem / 1024);
}
}
catch (Exception e)
{
Console.WriteLine("Unable to benchmark \"" + filename + "\":\nException occurred: " + e.Message
+ "\nStack trace:\n" + e.StackTrace);
}
}
}
}
public void SetBenchmarkMode(BenchmarkMode benchmarkMode)
{
//this.benchmarkMode = benchmarkMode;
//ringBuffer.SetBenchmarkMode(benchmarkMode);
}
private static DataTable Benchmark(BenchmarkMode mode, CaseSensitivity caseSensitivity)
{
TimeSpan duration;
CalculationEngine interpretedEngine = new CalculationEngine(CultureInfo.InvariantCulture, ExecutionMode.Interpreted, true, true, true);
CalculationEngine interpretedEngineCaseSensitive = new CalculationEngine(CultureInfo.InvariantCulture, ExecutionMode.Interpreted, true, true, false);
CalculationEngine compiledEngine = new CalculationEngine(CultureInfo.InvariantCulture, ExecutionMode.Compiled, true, true, true);
CalculationEngine compiledEngineCaseSensitive = new CalculationEngine(CultureInfo.InvariantCulture, ExecutionMode.Compiled, true, true, false);
BenchMarkOperation[] benchmarks =
{
new BenchMarkOperation()
{
Formula = "2+3*7", Mode = BenchmarkMode.Static, BenchMarkDelegate = BenchMarkCalculationEngine
},
new BenchMarkOperation()
{
Formula = "logn(var1, (2+3) * 500)", Mode = BenchmarkMode.SimpleFunction, BenchMarkDelegate = BenchMarkCalculationEngineFunctionBuild
},
new BenchMarkOperation()
{
Formula = "(var1 + var2 * 3)/(2+3) - something", Mode = BenchmarkMode.Simple, BenchMarkDelegate = BenchMarkCalculationEngineFunctionBuild
},
};
DataTable table = new DataTable();
table.Columns.Add("Engine");
table.Columns.Add("Case Sensitive");
table.Columns.Add("Formula");
table.Columns.Add("Iterations per Random Formula", typeof(int));
table.Columns.Add("Total Iteration", typeof(int));
table.Columns.Add("Total Duration");
foreach (BenchMarkOperation benchmark in benchmarks)
{
if (mode == BenchmarkMode.All || mode == benchmark.Mode)
{
if (caseSensitivity == CaseSensitivity.All || caseSensitivity == CaseSensitivity.CaseInSensitive)
{
duration = benchmark.BenchMarkDelegate(interpretedEngine, benchmark.Formula);
table.AddBenchmarkRecord("Interpreted", false, benchmark.Formula, null, NumberOfTests, duration);
}
if (caseSensitivity == CaseSensitivity.All || caseSensitivity == CaseSensitivity.CaseSensitive)
{
duration = benchmark.BenchMarkDelegate(interpretedEngineCaseSensitive, benchmark.Formula);
table.AddBenchmarkRecord("Interpreted", true, benchmark.Formula, null, NumberOfTests, duration);
}
if (caseSensitivity == CaseSensitivity.All || caseSensitivity == CaseSensitivity.CaseInSensitive)
{
duration = benchmark.BenchMarkDelegate(compiledEngine, benchmark.Formula);
table.AddBenchmarkRecord("Compiled", false, benchmark.Formula, null, NumberOfTests, duration);
}
if (caseSensitivity == CaseSensitivity.All || caseSensitivity == CaseSensitivity.CaseSensitive)
{
duration = benchmark.BenchMarkDelegate(compiledEngineCaseSensitive, benchmark.Formula);
table.AddBenchmarkRecord("Compiled", true, benchmark.Formula, null, NumberOfTests, duration);
}
}
}
if (mode == BenchmarkMode.All || mode == BenchmarkMode.Random)
{
List <string> functions = GenerateRandomFunctions(NumberOfFunctionsToGenerate);
if (caseSensitivity == CaseSensitivity.All || caseSensitivity == CaseSensitivity.CaseInSensitive)
{
//Interpreted Mode
duration = BenchMarkCalculationEngineRandomFunctionBuild(interpretedEngine, functions, NumberExecutionsPerRandomFunction);
table.AddBenchmarkRecord("Interpreted", false, string.Format("Random Mode {0} functions 3 variables", NumberOfFunctionsToGenerate),
NumberExecutionsPerRandomFunction, NumberExecutionsPerRandomFunction * NumberOfFunctionsToGenerate, duration);
}
if (caseSensitivity == CaseSensitivity.All || caseSensitivity == CaseSensitivity.CaseSensitive)
{
//Interpreted Mode(Case Sensitive)
duration = BenchMarkCalculationEngineRandomFunctionBuild(interpretedEngineCaseSensitive, functions, NumberExecutionsPerRandomFunction);
table.AddBenchmarkRecord("Interpreted", true, string.Format("Random Mode {0} functions 3 variables", NumberOfFunctionsToGenerate),
NumberExecutionsPerRandomFunction, NumberExecutionsPerRandomFunction * NumberOfFunctionsToGenerate, duration);
}
if (caseSensitivity == CaseSensitivity.All || caseSensitivity == CaseSensitivity.CaseInSensitive)
{
//Compiled Mode
duration = BenchMarkCalculationEngineRandomFunctionBuild(compiledEngine, functions, NumberExecutionsPerRandomFunction);
table.AddBenchmarkRecord("Compiled", false, string.Format("Random Mode {0} functions 3 variables", NumberOfFunctionsToGenerate),
NumberExecutionsPerRandomFunction, NumberExecutionsPerRandomFunction * NumberOfFunctionsToGenerate, duration);
}
if (caseSensitivity == CaseSensitivity.All || caseSensitivity == CaseSensitivity.CaseSensitive)
{
//Compiled Mode(Case Sensitive)
duration = BenchMarkCalculationEngineRandomFunctionBuild(compiledEngineCaseSensitive, functions, NumberExecutionsPerRandomFunction);
table.AddBenchmarkRecord("Compiled", true, string.Format("Random Mode {0} functions 3 variables", NumberOfFunctionsToGenerate),
NumberExecutionsPerRandomFunction, NumberExecutionsPerRandomFunction * NumberOfFunctionsToGenerate, duration);
}
}
return(table);
}
public void SetBenchmarkMode(BenchmarkMode benchmarkMode)
{
this.benchmarkMode = benchmarkMode;
}
private void view_StartBenchmark(object sender, EventArgs <BenchmarkMode> e)
{
IFrameProducer frameProducer = grabber as IFrameProducer;
//int width = grabber.Size.Width;
//int height = grabber.Size.Height;
//int depth = grabber.Depth;
int width = 2048;
int height = 1084;
int depth = 1;
benchmarkMode = e.Value;
switch (benchmarkMode)
{
case BenchmarkMode.Heartbeat:
case BenchmarkMode.Commitbeat:
case BenchmarkMode.Bradycardia:
case BenchmarkMode.FrameDrops:
break;
case BenchmarkMode.Noop:
consumers.Add(new ConsumerNoop());
break;
case BenchmarkMode.OccasionallySlow:
consumers.Add(new ConsumerOccasionallySlow());
break;
case BenchmarkMode.Slow:
consumers.Add(new ConsumerSlow());
break;
case BenchmarkMode.LZ4:
consumers.Add(new ConsumerLZ4());
break;
case BenchmarkMode.JPEG1:
consumers.Add(new ConsumerJPEG());
break;
case BenchmarkMode.FrameNumber:
consumers.Add(new ConsumerFrameNumber());
break;
default:
break;
}
// Start all consumer threads.
// In a real application some consumers would have been long started.
// for example the application main thread is also the display consumer.
foreach (IFrameConsumer consumer in consumers)
{
Thread thread = new Thread(consumer.Run)
{
IsBackground = true
};
thread.Name = consumer.GetType().Name;
thread.Start();
}
// Bind the ring buffer to consumers and producers.
pipeline = new FramePipeline(frameProducer, consumers, width, height, depth);
pipeline.SetBenchmarkMode(benchmarkMode);
grabber.Start();
// Activate all consumers.
// In a real application this may be done much later, in a "record" button event handler for example.
foreach (IFrameConsumer consumer in consumers)
{
consumer.Activate();
}
}
