private async Task <IEnumerable <ISerie <TKey, TEntry> > > WriteToDynamicStorageAsync(IDynamicStorage <TKey, TEntry> storage, IEnumerable <ISerie <TKey, TEntry> > series, bool useTemporaryStorageOnFailure)
{
var sw = Stopwatch.StartNew();
try
{
await storage.WriteAsync(series).ConfigureAwait(false);
_logger.Info($"Wrote {series.Sum( x => x.GetEntries().Count )} to dynamic storage. Elapsed = {sw.ElapsedMilliseconds} ms.");
return(series);
}
catch (Exception e1)
{
if (useTemporaryStorageOnFailure)
{
if (_temporaryStorage == null)
{
throw new InvalidOperationException("No temporary storage has been provided.");
}
try
{
_temporaryStorage.WriteAsync(series);
}
catch (Exception e2)
{
_logger.Error(e2, $"An error ocurred while writing to temporary storage after failing to write to dynamic storage. Elapsed = {sw.ElapsedMilliseconds} ms.");
}
}
_logger.Error(e1, $"An error ocurred while writing to dynamic storage. Elapsed = {sw.ElapsedMilliseconds} ms.");
return(_series);
}
}
public async Task MoveFromTemporaryStorageToPermanentStorageUntilCancelledAsync(int batchSize, TimeSpan movalInterval, CancellationToken cancellationToken = default(CancellationToken))
{
while (true)
{
try
{
await Task.Delay(movalInterval, cancellationToken).ConfigureAwait(false);
await MoveFromTemporaryStorageAsync(batchSize).ConfigureAwait(false);
}
catch (MissingTsdbServiceException)
{
throw;
}
catch (TaskCanceledException e) when(e.CancellationToken.IsCancellationRequested)
{
throw;
}
catch (Exception e)
{
_logger.Error(e, "An error ocurred while moving entries from temporary to permanent storage.");
}
}
}
internal async Task ConnectWithRetry()
{
while (true)
{
try
{
await ConnectToRedisAsync().ConfigureAwait(false);
// at this point, I know we are connected
var oldState = Interlocked.CompareExchange(ref _state, RedisConnectionState.Connected, RedisConnectionState.Closed);
if (oldState == RedisConnectionState.Closed)
{
_waitWhileDisconnected.SetResult(true);
}
else
{
Shutdown();
}
break;
}
catch (Exception e)
{
_logger.Error(e, "An error ocurred while connecting to redis.");
}
if (_state == RedisConnectionState.Disposing)
{
Shutdown();
break;
}
await Task.Delay(TimeSpan.FromSeconds(2)).ConfigureAwait(false);
}
}
private async void WriteLoop(CancellationToken cancel)
{
while (!_disposed)
{
var writes = GetBatchesToWrite();
if (writes != null)
{
try
{
await _client.WriteAsync(writes.Select(x => x.GetBatch()), _publishType, _publishMode, _useTempStorage).ConfigureAwait(false);
foreach (var write in writes)
{
write.Complete();
}
}
catch (Exception e)
{
_logger.Error(e, "An error ocurred while inserting batch of entries.");
foreach (var write in writes)
{
write.Fail(e);
}
}
}
try
{
if (_queuedCount < _maxBatchSize)
{
await Task.Delay(_writeInterval, _cts.Token).ConfigureAwait(false);
}
else if (_queuedCount >= _maxBatchSize * 10)
{
if (!_lastWarnTime.HasValue || (DateTime.UtcNow - _lastWarnTime) > MinWarningInterval)
{
_logger.Warn($"There are {_queuedCount} entries queued batches waiting to be written. This is an indication that the storage is too slow to handle the ingestion.");
_lastWarnTime = DateTime.UtcNow;
}
}
}
catch (OperationCanceledException)
{
// simply ignore
}
}
}
private async void MoveTemporaryData(DateTime timestamp)
{
if (!_disposed)
{
try
{
await _client.MoveFromTemporaryStorageAsync(_workProvider.GetTemporaryMovalBatchSize()).ConfigureAwait(false);
}
catch (Exception e)
{
_logger.Error(e, "An error ocurred while moving entries from temporary to dynamic storage.");
}
var scheduleTime = DateTime.UtcNow + _workProvider.GetTemporaryMovalInterval();
_unsubscribe = _scheduler.AddCommand(scheduleTime, MoveTemporaryData);
}
}
public Task SubscribeAsync <TKey, TEntry>(string id, IKeyConverter <TKey> keyConverter, SubscriptionType subscribe, Action <Serie <TKey, TEntry> > onMessage)
where TEntry : IRedisEntry, new()
{
if (_redisSubscriber == null)
{
throw new InvalidOperationException("The redis connection has not been started.");
}
var key = CreateSubscriptionKey(id, subscribe);
return(_redisSubscriber.SubscribeAsync(key, async(channel, data) =>
{
try
{
var entries = await RedisSerializer.Deserialize <TKey, TEntry>(data, keyConverter).ConfigureAwait(false);
onMessage(entries);
}
catch (Exception e)
{
_logger.Error(e, "An error ocurred while deserializing subscription message.");
}
}));
}