public T Retry <T>(Func <T> runTxFunc)
{
var exceptions = new List <Exception>();
var timer = new Stopwatch();
timer.Start();
var delayMs = _initialRetryDelayMs;
var counter = 0;
do
{
counter++;
try
{
return(runTxFunc());
}
catch (Exception e) when(e.IsRetriableError())
{
exceptions.Add(e);
var delay = TimeSpan.FromMilliseconds(ComputeDelayWithJitter(delayMs));
_logger?.Warn(e, $"Transaction failed and will be retried in {delay}ms.");
Thread.Sleep(delay);
delayMs = delayMs * _multiplier;
}
} while (timer.Elapsed.TotalMilliseconds < _maxRetryTimeMs);
timer.Stop();
throw new ServiceUnavailableException(
$"Failed after retried for {counter} times in {_maxRetryTimeMs}ms. " +
"Make sure that your database is online and retry again.", new AggregateException(exceptions));
}
public IObservable <T> Retry <T>(IObservable <T> work)
{
return(work.RetryWhen(failedWork =>
{
var handledExceptions = new List <Exception>();
var timer = Stopwatch.StartNew();
var delay = _initialDelay;
var retryCount = 1;
return failedWork.SelectMany(exc =>
{
if (!exc.IsRetriableError())
{
return Observable.Throw <int>(exc);
}
handledExceptions.Add(exc);
if (retryCount >= 2 && timer.ElapsedMilliseconds >= _maxRetryTimeout)
{
return Observable.Throw <int>(new ServiceUnavailableException(
$"Failed after retried for {retryCount} times in {_maxRetryTimeout} ms. " +
"Make sure that your database is online and retry again.",
new AggregateException(handledExceptions)));
}
var delayDuration = TimeSpan.FromMilliseconds(ComputeNextDelay(delay));
delay *= _delayMultiplier;
retryCount++;
_logger?.Warn(exc, $"Transaction failed and will be retried in {delay} ms.");
return Observable.Return(1).Delay(delayDuration);
});
}));
}
public void Send(IEnumerable <IRequestMessage> messages)
{
try
{
foreach (var message in messages)
{
Writer.Write(message);
LogDebug(MessagePattern, message);
}
Writer.Flush();
}
catch (Exception ex)
{
_logger?.Warn(ex, $"Unable to send message to server {_uri}, connection will be terminated.");
Stop();
throw;
}
}
public IRoutingTable UpdateRoutingTable(ISet <Uri> triedUris = null,
Func <IConnection, IRoutingTable> rediscoveryFunc = null)
{
rediscoveryFunc = rediscoveryFunc ?? Rediscovery;
var knownRouters = _routingTable.Routers;
foreach (var router in knownRouters)
{
triedUris?.Add(router);
IConnection conn = _poolManager.CreateClusterConnection(router);
if (conn == null)
{
_routingTable.Remove(router);
}
else
{
try
{
var roundRobinRoutingTable = rediscoveryFunc(conn);
if (!IsRoutingTableStale(roundRobinRoutingTable))
{
return(roundRobinRoutingTable);
}
}
catch (Exception e)
{
_logger?.Warn(e,
"Failed to update routing table with server uri={0}.", router);
if (e is SessionExpiredException)
{
// ignored
// Already handled by clusterConn.OnConnectionError to remove from load balancer
}
else
{
throw;
}
}
}
}
return(null);
}
public async Task SendAsync(IEnumerable <IRequestMessage> messages)
{
try
{
foreach (var message in messages)
{
Writer.Write(message);
LogDebug(MessagePattern, message);
}
await Writer.FlushAsync().ConfigureAwait(false);
}
catch (Exception ex)
{
_logger?.Warn(ex, $"Unable to send message to server {_uri}, connection will be terminated.");
await StopAsync().ConfigureAwait(false);
throw;
}
}
public async Task <T> RetryAsync <T>(Func <Task <T> > runTxAsyncFunc)
{
var exceptions = new List <Exception>();
var timer = new Stopwatch();
var delay = TimeSpan.Zero;
var delayMs = _initialRetryDelayMs;
var retryCount = 0;
var shouldRetry = false;
timer.Start();
do
{
retryCount++;
try
{
return(await runTxAsyncFunc().ConfigureAwait(false));
}
catch (Exception e) when(e.IsRetriableError())
{
exceptions.Add(e);
// we want the retry to happen at least twice and as much as the max retry time allows
shouldRetry = retryCount < 2 || timer.ElapsedMilliseconds < _maxRetryTimeMs;
if (shouldRetry)
{
delay = TimeSpan.FromMilliseconds(ComputeDelayWithJitter(delayMs));
_logger?.Warn(e, $"Transaction failed and will be retried in {delay} ms.");
await Task.Delay(delay).ConfigureAwait(false); // blocking for this delay
delayMs *= _multiplier;
}
}
} while (shouldRetry);
timer.Stop();
throw new ServiceUnavailableException(
$"Failed after retried for {retryCount} times in {_maxRetryTimeMs} ms. " +
"Make sure that your database is online and retry again.", new AggregateException(exceptions));
}
public async Task <IRoutingTable> UpdateRoutingTableAsync(ISet <Uri> triedUris = null,
Func <IConnection, Task <IRoutingTable> > rediscoveryFunc = null)
{
var knownRouters = _routingTable.Routers;
foreach (var router in knownRouters)
{
triedUris?.Add(router);
try
{
var conn = await _poolManager.CreateClusterConnectionAsync(router).ConfigureAwait(false);
if (conn == null)
{
_routingTable.Remove(router);
}
else
{
var newRoutingTable = await _discovery.DiscoverAsync(conn).ConfigureAwait(false);
if (!IsRoutingTableStale(newRoutingTable))
{
return(newRoutingTable);
}
}
}
catch (SecurityException e)
{
_logger?.Error(e,
"Failed to update routing table from server '{0}' because of a security exception.", router);
throw;
}
catch (Exception e)
{
_logger?.Warn(e, "Failed to update routing table from server '{0}'.", router);
}
}
return(null);
}
public T Retry <T>(Func <T> work)
{
var exceptions = new List <Exception>();
var delay = TimeSpan.Zero;
var delayMs = _initialDelay;
var retryCount = 0;
var shouldRetry = false;
var timer = Stopwatch.StartNew();
do
{
retryCount++;
try
{
return(work());
}
catch (Exception e) when(e.IsRetriableError())
{
exceptions.Add(e);
// we want the retry to happen at least twice and as much as the max retry time allows
shouldRetry = retryCount < 2 || timer.ElapsedMilliseconds < _maxRetryTimeout;
if (shouldRetry)
{
delay = TimeSpan.FromMilliseconds(ComputeNextDelay(delayMs));
_logger?.Warn(e, $"Transaction failed and will be retried in {delay}ms.");
Thread.Sleep(delay);
delayMs *= _delayMultiplier;
}
}
} while (shouldRetry);
timer.Stop();
throw new ServiceUnavailableException(
$"Failed after retried for {retryCount} times in {_maxRetryTimeout}ms. " +
"Make sure that your database is online and retry again.", new AggregateException(exceptions));
}
public void Warn(Exception cause, string message, params object[] args)
{
_delegate?.Warn(cause, Reformat(message), args);
}
