private async Task ProcessShardsAsync(IRecordProcessor processor)
{
try
{
var processShardsTask = new ConcurrentDictionary <Task <RecordResponse>, KShard>();
await WatchForShardUpdates(processShardsTask);
while (!_cancellationTokenSource.IsCancellationRequested)
{
if (processShardsTask.Count > 0)
{
var task = await Task.WhenAny(processShardsTask.Select(m => m.Key)).WithCancellation(_cancellationTokenSource.Token);
KShard shard;
if (task != null && task.IsCompleted && !task.IsCanceled && !task.IsFaulted)
{
Interlocked.Add(ref _currentRecordsProcessing, 1);
if (processShardsTask.TryRemove(task, out shard))
{
var record = await task;
if (record != null && record.GetRecordsResponse.Records.Any())
{
processor.Process(shard.ShardId, record.GetRecordsResponse.Records.LastOrDefault().SequenceNumber, shard.LastUpdateUtc, record.GetRecordsResponse.Records, SaveCheckpoint);
if (record.GetRecordsResponse.NextShardIterator != null)
{
shard.UpdateShardInformation(record.GetRecordsResponse);
var getRecordsTask = GetRecordResponse(shard, record.CancellationToken);
processShardsTask.TryAdd(getRecordsTask, shard);
}
}
}
Interlocked.Decrement(ref _currentRecordsProcessing);
}
else //the task was cancelled or faulted or there is some error
{
if (task != null)
{
processShardsTask.TryRemove(task, out shard);
}
}
}
else
{
await Task.Delay(2000); //sleep and hope that we get some cool shards to work with
}
}
}
catch (OperationCanceledException e)
{
Log.Debug(e, "ProcessShardsAsync was cancelled");
}
}
