/// <summary>
/// Main method for the sample.
/// </summary>
/// <param name="args">command line arguments.</param>
public static async Task Main(string[] args)
{
try
{
BenchmarkConfig config = BenchmarkConfig.From(args);
await Program.AddAzureInfoToRunSummary();
ThreadPool.SetMinThreads(config.MinThreadPoolSize, config.MinThreadPoolSize);
if (config.EnableLatencyPercentiles)
{
TelemetrySpan.IncludePercentile = true;
TelemetrySpan.ResetLatencyHistogram(config.ItemCount);
}
config.Print();
Program program = new Program();
RunSummary runSummary = await program.ExecuteAsync(config);
}
finally
{
Console.WriteLine($"{nameof(CosmosBenchmark)} completed successfully.");
if (Debugger.IsAttached)
{
Console.WriteLine("Press any key to exit...");
Console.ReadLine();
}
}
}
public async Task ExecuteAsync(
int iterationCount,
bool isWarmup,
Action completionCallback)
{
Trace.TraceInformation($"Executor {this.executorId} started");
try
{
int currentIterationCount = 0;
do
{
OperationResult?operationResult = null;
await this.operation.Prepare();
using (TelemetrySpan telemetrySpan = TelemetrySpan.StartNew(
() => operationResult.Value,
disableTelemetry: isWarmup))
{
try
{
operationResult = await this.operation.ExecuteOnceAsync();
// Success case
this.SuccessOperationCount++;
this.TotalRuCharges += operationResult.Value.RuCharges;
}
catch (Exception ex)
{
// failure case
this.FailedOperationCount++;
// Special case of cosmos exception
double opCharge = 0;
if (ex is CosmosException cosmosException)
{
opCharge = cosmosException.RequestCharge;
this.TotalRuCharges += opCharge;
}
operationResult = new OperationResult()
{
// TODO: Populate account, databse, collection context into ComsosDiagnostics
RuCharges = opCharge,
lazyDiagnostics = () => ex.ToString(),
};
}
}
currentIterationCount++;
} while (currentIterationCount < iterationCount);
Trace.TraceInformation($"Executor {this.executorId} completed");
}
finally
{
completionCallback();
}
}
public static TelemetrySpan StartNew(
Func <OperationResult> lazyOperationResult,
bool disableTelemetry)
{
TelemetrySpan span = new TelemetrySpan();
span.stopwatch = Stopwatch.StartNew();
span.lazyOperationResult = lazyOperationResult;
span.disableTelemetry = disableTelemetry;
return(span);
}
private async Task <RunSummary> LogOutputStats(
BenchmarkConfig benchmarkConfig,
IExecutor[] executors)
{
const int outputLoopDelayInSeconds = 1;
IList <int> perLoopCounters = new List <int>();
Summary lastSummary = new Summary();
Stopwatch watch = new Stopwatch();
watch.Start();
bool isLastIterationCompleted = false;
do
{
isLastIterationCompleted = this.pendingExecutorCount <= 0;
Summary currentTotalSummary = new Summary();
for (int i = 0; i < executors.Length; i++)
{
IExecutor executor = executors[i];
Summary executorSummary = new Summary()
{
successfulOpsCount = executor.SuccessOperationCount,
failedOpsCount = executor.FailedOperationCount,
ruCharges = executor.TotalRuCharges,
};
currentTotalSummary += executorSummary;
}
// In-theory summary might be lower than real as its not transactional on time
currentTotalSummary.elapsedMs = watch.Elapsed.TotalMilliseconds;
Summary diff = currentTotalSummary - lastSummary;
lastSummary = currentTotalSummary;
diff.Print(currentTotalSummary.failedOpsCount + currentTotalSummary.successfulOpsCount);
perLoopCounters.Add((int)diff.Rps());
await Task.Delay(TimeSpan.FromSeconds(outputLoopDelayInSeconds));
}while (!isLastIterationCompleted);
using (ConsoleColorContext ct = new ConsoleColorContext(ConsoleColor.Green))
{
Console.WriteLine();
Console.WriteLine("Summary:");
Console.WriteLine("--------------------------------------------------------------------- ");
lastSummary.Print(lastSummary.failedOpsCount + lastSummary.successfulOpsCount);
// Skip first 5 and last 5 counters as outliers
IEnumerable <int> exceptFirst5 = perLoopCounters.Skip(5);
int[] summaryCounters = exceptFirst5.Take(exceptFirst5.Count() - 5).OrderByDescending(e => e).ToArray();
RunSummary runSummary = new RunSummary(
benchmarkConfig,
executors.Length);
if (summaryCounters.Length > 10)
{
Console.WriteLine();
Utility.TeeTraceInformation("After Excluding outliers");
runSummary.Top10PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.1 * summaryCounters.Length)).Average(), 0);
runSummary.Top20PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.2 * summaryCounters.Length)).Average(), 0);
runSummary.Top30PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.3 * summaryCounters.Length)).Average(), 0);
runSummary.Top40PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.4 * summaryCounters.Length)).Average(), 0);
runSummary.Top50PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.5 * summaryCounters.Length)).Average(), 0);
runSummary.Top60PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.6 * summaryCounters.Length)).Average(), 0);
runSummary.Top70PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.7 * summaryCounters.Length)).Average(), 0);
runSummary.Top80PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.8 * summaryCounters.Length)).Average(), 0);
runSummary.Top90PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.9 * summaryCounters.Length)).Average(), 0);
runSummary.Top95PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.95 * summaryCounters.Length)).Average(), 0);
runSummary.Top99PercentAverageRps = Math.Round(summaryCounters.Take((int)(0.99 * summaryCounters.Length)).Average(), 0);
runSummary.AverageRps = Math.Round(summaryCounters.Average(), 0);
runSummary.Top50PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(50);
runSummary.Top75PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(75);
runSummary.Top90PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(90);
runSummary.Top95PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(95);
runSummary.Top98PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(98);
runSummary.Top99PercentLatencyInMs = TelemetrySpan.GetLatencyPercentile(99);
runSummary.MaxLatencyInMs = TelemetrySpan.GetLatencyPercentile(100);
string summary = JsonConvert.SerializeObject(runSummary);
Utility.TeeTraceInformation(summary);
}
else
{
Utility.TeeTraceInformation("Please adjust ItemCount high to run of at-least 1M");
}
Console.WriteLine("--------------------------------------------------------------------- ");
return(runSummary);
}
}