private Task WatchForShardUpdates(ConcurrentDictionary <Task <RecordResponse>, KShard> processShardsTask)
{
Task.Run(async() =>
{
var getRecordCancellationToken = new CancellationTokenSource();
while (!_cancellationTokenSource.IsCancellationRequested)
{
var newActiveShards = await _utilities.GetActiveShardsAsync();
if (newActiveShards.Count != processShardsTask.Count && Interlocked.CompareExchange(ref _currentRecordsProcessing, 0, 0) == 0)
{
if (processShardsTask.Count > 0) //this is a bit ugly but it'll do - seems robust enough for now
{
getRecordCancellationToken.Cancel(); //cancel existing record tasks
getRecordCancellationToken = new CancellationTokenSource(); //create a new token for any future cancellations
}
var shards = await _dynamoDb.GetShards(newActiveShards.Select(m => m.ShardId).ToList());
//Add new tasks to be processed
foreach (var shard in shards)
{
await GenerateShardIteratorRequest(shard);
processShardsTask.TryAdd(GetRecordResponse(shard, getRecordCancellationToken.Token), shard);
}
Log.Information("Shards Found: {0}", shards.Count);
}
await Task.Delay(TimeSpan.FromMinutes(2));
}
}, _cancellationTokenSource.Token);
return(Task.FromResult(0));
}
