private async Task CreateWorkerAsync(Func <PerformanceData, Task> workload, int taskId, CancellationToken cancellationToken)
{
var timer = new Stopwatch();
var retry = new RetryHandler();
var random = new Random();
// Non-conflicting keys based on task id.
int minKey = taskId * _keysPerTask;
int maxKey = (taskId + 1) * _keysPerTask;
while (!cancellationToken.IsCancellationRequested)
{
// Generate a unique, non-conflicting write for this workload invocation.
long key = random.Next(minKey, maxKey);
var data = RandomGenerator.GetPerformanceData(key.ToString());
timer.Restart();
try
{
// Invoke the workload.
await workload.Invoke(data).ConfigureAwait(false);
timer.Stop();
retry.Reset();
lock (_latency)
{
_latency.AddSample(timer.Elapsed.TotalMilliseconds);
}
// Delay between workload invocations.
int delayInMs = random.Next(_minWorkloadDelayInMs, _maxWorkloadDelayInMs);
await Task.Delay(delayInMs, cancellationToken).ConfigureAwait(false);
}
catch (Exception e)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
timer.Stop();
var retryAfter = retry.Retry();
// Track metrics.
Interlocked.Increment(ref _errors);
lock (_latency)
{
_latency.AddSample(timer.Elapsed.TotalMilliseconds);
}
// Exponential delay after exceptions.
_logger.LogError(e, "Unexpected exception {ExceptionType} in {WorkloadName}. Retrying in {RetryInMs} ms.", e.GetType(), _workloadName, retryAfter.TotalMilliseconds);
await Task.Delay(retryAfter, cancellationToken).ConfigureAwait(false);
}
}
}
private async Task CreateWorkerAsync(Func <Task <long> > workload, int taskId, CancellationToken cancellationToken)
{
var timer = new Stopwatch();
var retry = new RetryHandler();
while (!cancellationToken.IsCancellationRequested)
{
timer.Restart();
try
{
// Invoke the workload.
long operations = await workload.Invoke().ConfigureAwait(false);
timer.Stop();
retry.Reset();
// Track metrics.
Interlocked.Add(ref _operations, operations);
Interlocked.Add(ref _latency, timer.ElapsedMilliseconds);
}
catch (Exception e)
{
timer.Stop();
if (cancellationToken.IsCancellationRequested)
{
return;
}
// Check if this is an exception indicating we are being throttled.
var throttle = _throttle.Invoke(e);
if (throttle != null)
{
await Task.Delay(throttle.Value, cancellationToken).ConfigureAwait(false);
}
else
{
var retryAfter = retry.Retry();
// Track metrics.
Interlocked.Add(ref _latency, timer.ElapsedMilliseconds);
Interlocked.Increment(ref _errors);
// Exponential delay after exceptions.
_logger.LogError(e, "Unexpected exception {ExceptionType} in {WorkloadName}. Retrying in {RetryInMs} ms.", e.GetType(), _workloadName, retryAfter.TotalMilliseconds);
await Task.Delay(retryAfter, cancellationToken).ConfigureAwait(false);
}
}
}
}