/// <summary>
/// Performs the individual bench mark. This is a benchmark which fetches all SalesOrderHeader elements with the keys specified, individually.
/// </summary>
/// <param name="keys">The keys for all elements to fetch.</param>
/// <param name="discardResults">if set to <c>true</c> the results are returned but are not collected.</param>
/// <returns>
/// A filled in benchmark result object, with the total time taken to fetch all elements of the keys specified. EnumerationTime is not set. Number of
/// </returns>
public BenchResult PerformIndividualBenchMark(List <int> keys, bool discardResults)
{
var toReturn = new BenchResult();
int numberOfElementsFetched = 0;
var sw = new Stopwatch();
sw.Start();
foreach (var key in keys)
{
var element = FetchIndividual(key);
var verifyResult = VerifyElement(element);
if (verifyResult > 0)
{
numberOfElementsFetched++;
}
}
sw.Stop();
toReturn.FetchTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
toReturn.IncNumberOfRowsForType(typeof(T), numberOfElementsFetched);
if (!discardResults)
{
_individualBenchmarkResults.Add(toReturn);
}
return(toReturn);
}
/// <summary>
/// Performs the eager load benchmark, asynchronously. This is a benchmark which will fetch a 2-edge graph of 1000 sales order headers, all related customer entities and all related
/// sales order detail entities. It will use a Task.Run to perform the async code.
/// </summary>
/// <param name="discardResults">if set to <c>true</c> the results are returned but are not collected.</param>
/// <returns>
/// A filled in benchmark result object
/// </returns>
public BenchResult PerformAsyncEagerLoadBenchmark(bool discardResults)
{
var toReturn = new BenchResult();
var sw = new Stopwatch();
long memoryBeforeRun = 0;
if (AppDomain.MonitoringIsEnabled)
{
memoryBeforeRun = AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize;
}
sw.Start();
IEnumerable <T> headers = null;
Task.Run(async() => headers = await FetchGraphAsync()).GetAwaiter().GetResult();
sw.Stop();
if (AppDomain.MonitoringIsEnabled)
{
toReturn.NumberOfBytesAllocated = AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize - memoryBeforeRun;
}
toReturn.FetchTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
sw.Reset();
sw.Start();
VerifyGraph(headers, toReturn);
sw.Stop();
if (!discardResults)
{
_asyncEagerLoadBenchmarkResults.Add(toReturn);
}
return(toReturn);
}
protected BenchResult[] BenchInParallel <TBench, TBenchResult>(
Func <TBench> buildFunc, Func <TBench, TBenchResult> benchFunc,
Action <TBenchResult, BenchResult> setBenchResultFunc)
{
var benchs = Enumerable.Range(0, Environment.ProcessorCount)
.Select(i => buildFunc()).ToArray();
var results = new BenchResult[Environment.ProcessorCount];
var sw = new Stopwatch();
sw.Start();
var tasks = Enumerable.Range(0, Environment.ProcessorCount)
.Select(i => Task.Run(() => {
var swi = new Stopwatch();
sw.Start();
var result = benchFunc(benchs[i]);
results[i] = BuildResult(sw);
setBenchResultFunc(result, results[i]);
})).ToArray();
Task.WaitAll(tasks);
return(results);
}
protected override BenchResult[] BenchInParallel <TBench, TBenchResult>(
Func <TBench> buildFunc, Func <TBench, TBenchResult> benchFunc,
Action <TBenchResult, BenchResult> setBenchResultFunc)
{
var benchs = Enumerable.Range(0, Environment.ProcessorCount)
.Select(i => buildFunc()).ToArray();
var results = new BenchResult[Environment.ProcessorCount];
var sw = new Stopwatch();
sw.Start();
var tasks = Enumerable.Range(0, Environment.ProcessorCount)
.Select(i => Task.Run(() => {
var swi = new Stopwatch();
sw.Start();
var callTime = DoCallBench();
results[i] = BuildResult(sw);
results[i].Elapsed = TimeSpan.FromMilliseconds(callTime);
})).ToArray();
Task.WaitAll(tasks);
return(results);
}
/// <summary>
/// Performs the set benchmark. This is a benchmark which fetches the full set of sales order headers, enumerates it and returns the times it took
/// to perform these actions as well as the number of rows read.
/// </summary>
/// <param name="discardResults">if set to <c>true</c> the results are returned but are not collected.</param>
/// <returns>
/// A filled in benchmark result object
/// </returns>
public BenchResult PerformSetBenchmark(bool discardResults)
{
var toReturn = new BenchResult();
var sw = new Stopwatch();
long memoryBeforeRun = 0;
if (AppDomain.MonitoringIsEnabled)
{
memoryBeforeRun = AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize;
}
sw.Start();
var headers = FetchSet();
sw.Stop();
if (AppDomain.MonitoringIsEnabled)
{
toReturn.NumberOfBytesAllocated = AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize - memoryBeforeRun;
}
toReturn.FetchTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
sw.Reset();
sw.Start();
toReturn.IncNumberOfRowsForType(typeof(T), VerifyData(headers));
sw.Stop();
toReturn.EnumerationTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
if (!discardResults)
{
_setBenchmarkResults.Add(toReturn);
}
return(toReturn);
}
public virtual BenchResult PopulateResult(BenchResult benchResult, TResult dhrystoneResult)
{
if (dhrystoneResult is BenchResult[] results)
{
return(BuildParallelResult(benchResult, results));
}
return(benchResult);
}
private static void WriteResult(BenchResult benchResult)
{
writer.WriteTitle("{0,-30}", benchResult.BenchmarkName);
writer.WriteValue("{0,15} ms", string.Format("{0:F2}", benchResult.Elapsed.TotalMilliseconds));
writer.WriteValue("{0,15} pts", string.Format("{0:F2}", benchResult.Points));
writer.WriteValue("{0,15} {1}", string.Format("{0:F2}", benchResult.Result), benchResult.Units);
writer.WriteLine();
}
public override BenchResult PopulateResult(BenchResult benchResult, LinpackResult linpackResult)
{
benchResult.Result = linpackResult.MFLOPS;
benchResult.Points = linpackResult.MFLOPS * Ratio;
benchResult.Units = "MFLOPS";
benchResult.Output = linpackResult.Output;
return(benchResult);
}
public override BenchResult PopulateResult(BenchResult benchResult, WhetstoneResult dhrystoneResult)
{
benchResult.Result = dhrystoneResult.MWIPS;
benchResult.Points = Convert.ToDecimal(dhrystoneResult.MWIPS);
benchResult.Units = "MWIPS";
benchResult.Output = dhrystoneResult.Output;
return(benchResult);
}
public override BenchResult PopulateResult(BenchResult benchResult, MemoryBenchmarkResult result)
{
benchResult.Points = result.Average * Ratio;
benchResult.Result = result.Average;
benchResult.Units = "MB/s";
benchResult.Output = result.Output;
return(benchResult);
}
public override BenchResult PopulateResult(BenchResult benchResult, MemoryBenchmarkResult result)
{
benchResult.Points = Convert.ToDecimal(result.Average * Ratio);
benchResult.Result = Convert.ToDecimal(result.Average);
benchResult.Units = "MB/s";
benchResult.Output = result.Output;
return(benchResult);
}
public override BenchResult PopulateResult(BenchResult benchResult, DhrystoneResult dhrystoneResult)
{
benchResult.Result = dhrystoneResult.VaxMips;
benchResult.Points = dhrystoneResult.VaxMips * Ratio;
benchResult.Units = "DMIPS";
benchResult.Output = dhrystoneResult.Output;
return(benchResult);
}
public override BenchResult PopulateResult(BenchResult benchResult, Scimark2.Scimark2Result dhrystoneResult)
{
benchResult.Result = dhrystoneResult.CompositeScore;
benchResult.Points = dhrystoneResult.CompositeScore * Ratio;
benchResult.Units = "CompositeScore";
benchResult.Output = dhrystoneResult.Output;
return(benchResult);
}
public override BenchResult PopulateResult(BenchResult benchResult, WhetstoneResult whetstoneResult)
{
benchResult.Result = whetstoneResult.MWIPS;
benchResult.Points = whetstoneResult.MWIPS * Ratio;
benchResult.Units = "MWIPS";
benchResult.Output = whetstoneResult.Output;
return(benchResult);
}
public override BenchResult PopulateResult(BenchResult benchResult, BenchResult[] dhrystoneResult)
{
var result = BuildParallelResult(benchResult, dhrystoneResult);
result.Result = dhrystoneResult.Sum(r => r.Result);
result.Units = "DMIPS";
result.Output = string.Concat(dhrystoneResult.Select(s => s.Output));
return(result);
}
static void PrintAverageResult(List <BenchResult> results)
{
if (results.Count == 1)
{
return;
}
Console.WriteLine("Average: ");
BenchResult.Average(results).Print();
}
protected void CheckResult(BenchResult result)
{
Assert.True(result != null);
Assert.True(result.Connections == _connections);
Assert.True(result.Items.Length > 0);
Assert.True(result.Items[0].SendingStep == _sending);
Assert.True(result.Items[0].Message.TotalSend > 0);
Assert.True(result.Items[0].Message.TotalRecv > 0);
}
public override BenchResult PopulateResult(BenchResult benchResult, BenchResult[] scimark2Result)
{
var result = BuildParallelResult(benchResult, scimark2Result);
result.Result = scimark2Result.Sum(r => r.Result);
result.Units = "CompositeScore";
result.Output = string.Concat(scimark2Result.Select(s => s.Output));
return(result);
}
private bool VerifyGraphElement(T parent, BenchResult resultContainer)
{
var toReturn = VerifyElement(parent);
if (toReturn <= 0)
{
return(false);
}
VerifyGraphElementChildren(parent, resultContainer);
return(resultContainer.NumberOfRowsFetchedPerType.Count == 2 && resultContainer.NumberOfRowsFetchedPerType.All(kvp => kvp.Value > 0));
}
public override BenchResult PopulateResult(BenchResult benchResult, BenchResult[] dhrystoneResult)
{
var result = BuildParallelResult(benchResult, dhrystoneResult);
var sum = dhrystoneResult.Sum(r => Convert.ToDouble(r.Result));
result.Result = sum;
result.Points = sum * Ratio;
result.Units = "MB/s";
result.Output = string.Concat(dhrystoneResult.Select(s => s.Output));
return(result);
}
/// <summary>
/// Performs the insert set benchmark. This is a benchmark which inserts a set of entities in one go, collects the time it took to do so, verifies all
/// rows inserted and deletes the rows afterwards.
/// </summary>
/// <param name="amountToInsert">the amount of entities to insert</param>
/// <param name="batchSize">The size of the batch to use as maximum, if supported</param>
/// <param name="discardResults">if set to <c>true</c> the results are returned but are not collected.</param>
/// <returns>
/// A filled in benchmark result object
/// </returns>
public BenchResult PerformInsertSetBenchmark(int amountToInsert, int batchSize, bool discardResults)
{
if (_insertedIdRetriever == null)
{
throw new InvalidOperationException($"The func to retrieve the id of an instance of '{typeof(TInsert).FullName}' hasn't been specified, data can't be verified, exiting.");
}
var toReturn = new BenchResult()
{
InsertBatchSize = batchSize
};
var sw = new Stopwatch();
long memoryBeforeRun = 0;
// account for the entities to insert as well in the memory to collect as it's part of the process.
if (CollectMemoryAllocated)
{
#if NETCOREAPP
memoryBeforeRun = GC.GetAllocatedBytesForCurrentThread();
#else
memoryBeforeRun = AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize;
#endif
}
var setToInsert = CreateSetForInserts(amountToInsert);
if (setToInsert.Count() != amountToInsert)
{
throw new InvalidOperationException($"The method 'CreateSetForInserts' returns a set of {setToInsert.Count()} elements while {amountToInsert} were requested, exiting");
}
sw.Start();
InsertSet(setToInsert, batchSize);
sw.Stop();
if (CollectMemoryAllocated)
{
#if NETCOREAPP
toReturn.NumberOfBytesAllocated = GC.GetAllocatedBytesForCurrentThread() - memoryBeforeRun;
#else
toReturn.NumberOfBytesAllocated = AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize - memoryBeforeRun;
#endif
}
toReturn.ActionTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
var setFetchedFromInserted = FetchInserted(amountToInsert);
toReturn.IncNumberOfRowsForType(typeof(TInsert), VerifyInserted(setToInsert, setFetchedFromInserted));
if (!discardResults)
{
_setInsertBenchmarkResults.Add(toReturn);
}
DeleteInserted(setToInsert);
return(toReturn);
}
private void VerifyGraph(IEnumerable <T> toEnumerate, BenchResult resultContainer)
{
int amount = 0;
// parents
foreach (var v in toEnumerate)
{
if (!VerifyGraphElement(v, resultContainer))
{
// parent not loaded, fail complete graph
return;
}
amount++;
}
resultContainer.IncNumberOfRowsForType(typeof(T), amount);
}
static void BenchmarkWithConfig(int repeat, YcsbConfig config)
{
List <BenchResult> results = new List <BenchResult>(repeat);
for (int i = 0; i < repeat; ++i)
{
if (repeat > 1)
{
Console.WriteLine($"ROUND {i + 1}:");
}
BenchResult result = BenchmarkWithConfigOnce(config);
results.Add(result);
result.Print();
}
PrintAverageResult(results);
Console.WriteLine("---");
}
/// <summary>
/// Performs the individual bench mark. This is a benchmark which fetches all SalesOrderHeader elements with the keys specified, individually.
/// </summary>
/// <param name="keys">The keys for all elements to fetch.</param>
/// <param name="discardResults">if set to <c>true</c> the results are returned but are not collected.</param>
/// <returns>
/// A filled in benchmark result object, with the total time taken to fetch all elements of the keys specified. EnumerationTime is not set. Number of
/// </returns>
public BenchResult PerformIndividualBenchMark(List <int> keys, bool discardResults)
{
var toReturn = new BenchResult();
int numberOfElementsFetched = 0;
var sw = new Stopwatch();
long memoryBeforeRun = 0;
if (CollectMemoryAllocated)
{
#if NETCOREAPP
memoryBeforeRun = GC.GetAllocatedBytesForCurrentThread();
#else
memoryBeforeRun = AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize;
#endif
}
sw.Start();
bool first = true;
foreach (var key in keys)
{
var element = FetchIndividual(key);
if (first && CollectMemoryAllocated)
{
// only the first iteration is interesting.
#if NETCOREAPP
toReturn.NumberOfBytesAllocated = GC.GetAllocatedBytesForCurrentThread() - memoryBeforeRun;
#else
toReturn.NumberOfBytesAllocated = AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize - memoryBeforeRun;
#endif
first = false;
}
var verifyResult = VerifyElement(element);
if (verifyResult > 0)
{
numberOfElementsFetched++;
}
}
sw.Stop();
toReturn.ActionTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
toReturn.IncNumberOfRowsForType(typeof(TFetch), numberOfElementsFetched);
if (!discardResults)
{
_individualBenchmarkResults.Add(toReturn);
}
return(toReturn);
}
/// <summary>
/// Performs the set benchmark. This is a benchmark which fetches the full set of sales order headers, enumerates it and returns the times it took
/// to perform these actions as well as the number of rows read.
/// </summary>
/// <returns>A filled in benchmark result object</returns>
public BenchResult PerformSetBenchmark()
{
var toReturn = new BenchResult();
var sw = new Stopwatch();
sw.Start();
var headers = FetchSet();
sw.Stop();
toReturn.FetchTimeInMilliseconds = sw.ElapsedMilliseconds;
sw.Reset();
sw.Start();
toReturn.NumberOfRowsFetched = VerifyData(headers);
sw.Stop();
toReturn.EnumerationTimeInMilliseconds = sw.ElapsedMilliseconds;
_setBenchmarkResults.Add(toReturn);
return(toReturn);
}
static public BenchResult Average(IEnumerable <BenchResult> results)
{
int c = results.Count();
BenchResult avg = new BenchResult();
foreach (var result in results)
{
avg.NAbort += result.NAbort;
avg.NCommit += result.NCommit;
avg.CompleteTime += result.CompleteTime;
avg.SuggestedThroughput += result.SuggestedThroughput;
}
avg.NAbort /= c;
avg.NCommit /= c;
avg.SuggestedThroughput /= c;
avg.CompleteTime = new TimeSpan(avg.CompleteTime.Ticks / c);
return(avg);
}
/// <summary>
/// Performs the eager load benchmark. This is a benchmark which will fetch a 2-edge graph of 1000 sales order headers, all related customer entities and all related
/// sales order detail entities.
/// </summary>
/// <param name="discardResults">if set to <c>true</c> the results are returned but are not collected.</param>
/// <returns>
/// A filled in benchmark result object
/// </returns>
public BenchResult PerformEagerLoadBenchmark(bool discardResults)
{
var toReturn = new BenchResult();
var sw = new Stopwatch();
sw.Start();
var headers = FetchGraph();
sw.Stop();
toReturn.FetchTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
sw.Reset();
sw.Start();
VerifyGraph(headers, toReturn);
sw.Stop();
if (!discardResults)
{
_eagerLoadBenchmakResults.Add(toReturn);
}
return(toReturn);
}
/// <summary>
/// Performs the eager load benchmark, asynchronously. This is a benchmark which will fetch a 2-edge graph of 1000 sales order headers, all related customer entities and all related
/// sales order detail entities. It will use a Task.Run to perform the async code.
/// </summary>
/// <param name="discardResults">if set to <c>true</c> the results are returned but are not collected.</param>
/// <returns>
/// A filled in benchmark result object
/// </returns>
public BenchResult PerformAsyncEagerLoadBenchmark(bool discardResults)
{
var toReturn = new BenchResult();
var sw = new Stopwatch();
sw.Start();
IEnumerable <T> headers = null;
Task.Run(async() => headers = await FetchGraphAsync()).GetAwaiter().GetResult();
sw.Stop();
toReturn.FetchTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
sw.Reset();
sw.Start();
VerifyGraph(headers, toReturn);
sw.Stop();
if (!discardResults)
{
_asyncEagerLoadBenchmarkResults.Add(toReturn);
}
return(toReturn);
}
/// <summary>
/// Performs the set benchmark. This is a benchmark which fetches the full set of sales order headers, enumerates it and returns the times it took
/// to perform these actions as well as the number of rows read.
/// </summary>
/// <param name="discardResults">if set to <c>true</c> the results are returned but are not collected.</param>
/// <returns>
/// A filled in benchmark result object
/// </returns>
public BenchResult PerformSetBenchmark(bool discardResults)
{
var toReturn = new BenchResult();
var sw = new Stopwatch();
sw.Start();
var headers = FetchSet();
sw.Stop();
toReturn.FetchTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
sw.Reset();
sw.Start();
toReturn.IncNumberOfRowsForType(typeof(T), VerifyData(headers));
sw.Stop();
toReturn.EnumerationTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
if (!discardResults)
{
_setBenchmarkResults.Add(toReturn);
}
return(toReturn);
}
/// <summary>
/// Performs the eager load benchmark, asynchronously. This is a benchmark which will fetch a 2-edge graph of 1000 sales order headers, all related customer entities and all related
/// sales order detail entities. It will use a Task.Run to perform the async code.
/// </summary>
/// <param name="discardResults">if set to <c>true</c> the results are returned but are not collected.</param>
/// <returns>
/// A filled in benchmark result object
/// </returns>
public BenchResult PerformAsyncEagerLoadBenchmark(bool discardResults)
{
var toReturn = new BenchResult();
var sw = new Stopwatch();
long memoryBeforeRun = 0;
if (CollectMemoryAllocated)
{
#if NETCOREAPP
memoryBeforeRun = GC.GetAllocatedBytesForCurrentThread();
#else
memoryBeforeRun = AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize;
#endif
}
sw.Start();
IEnumerable <TFetch> headers = null;
Task.Run(async() => headers = await FetchGraphAsync()).GetAwaiter().GetResult();
sw.Stop();
if (CollectMemoryAllocated)
{
#if NETCOREAPP
// Currently not working (As of .NET Core 2.1, reports 0 bytes, as this method doesn't take into account allocs on other threads)
// there's no equivalent yet available.
toReturn.NumberOfBytesAllocated = GC.GetAllocatedBytesForCurrentThread() - memoryBeforeRun;
#else
toReturn.NumberOfBytesAllocated = AppDomain.CurrentDomain.MonitoringTotalAllocatedMemorySize - memoryBeforeRun;
#endif
}
toReturn.ActionTimeInMilliseconds = sw.Elapsed.TotalMilliseconds;
sw.Reset();
sw.Start();
VerifyGraph(headers, toReturn);
sw.Stop();
if (!discardResults)
{
_asyncEagerLoadBenchmarkResults.Add(toReturn);
}
return(toReturn);
}
