private async Task Recover()
{
RecoveryEnabledChannel[] channelsToRecover;
lock (_channelRecoveries)
channelsToRecover = _channelRecoveries.ToArray();
foreach (var recoveryEnabledChannel in channelsToRecover)
{
if (LogAdapter.ExtendedLogEnabled)
{
LogAdapter.LogWarn(LogSource, "Recover: recovering channel " + recoveryEnabledChannel.ChannelNumber);
}
try
{
await recoveryEnabledChannel.DoRecover(_connection).ConfigureAwait(false);
}
catch (Exception ex)
{
if (LogAdapter.IsErrorEnabled)
{
LogAdapter.LogError(LogSource, "Recover: error recovering channel " + recoveryEnabledChannel.ChannelNumber, ex);
}
throw;
}
}
}
// Runs from a background thread
private async Task <bool> CycleReconnect()
{
var hosts = _hostnames.ToArray();
var index = 0;
var firstRun = true;
// TODO: maxattempts or api hook to allow continuing/stopping
while (!_recoveryCancellationTokenSource.Token.IsCancellationRequested)
{
var hostToTry = hosts[index++ % hosts.Length];
if (hostToTry == _selectedHostname && firstRun) // skip the same hostname, but only once
{
firstRun = false;
continue;
}
if (LogAdapter.IsWarningEnabled)
{
LogAdapter.LogWarn(LogSource, "Trying to re-connect to " + hostToTry);
}
var succeeded = await _connection.InternalConnect(hostToTry, throwOnError : false).ConfigureAwait(false);
if (succeeded)
{
return(true);
}
// TODO: config/parameter for wait time
Thread.Sleep(1000);
}
return(false);
}
public void Read_ChannelFlow(Action <bool> continuation)
{
bool isActive = _amqpReader.ReadBits() != 0;
LogAdapter.LogWarn(LogSource, "< ChannelFlow " + isActive);
continuation(isActive);
}
private void TryInitiateRecovery()
{
if (Interlocked.CompareExchange(ref _inRecovery, 1, 0) == 0)
{
if (LogAdapter.IsWarningEnabled)
{
LogAdapter.LogWarn(LogSource, "TryInitiateRecovery starting recovery process");
}
// block all channels
lock (_channelRecoveries)
foreach (var recoveryEnabledChannel in _channelRecoveries)
{
recoveryEnabledChannel.Disconnected();
}
// from this point on, no api calls are allowed on the channel decorators
ThreadFactory.BackgroundThread(async(pthis) =>
{
try
{
if (LogAdapter.IsWarningEnabled)
{
LogAdapter.LogWarn(LogSource, "Starting Recovery. Waiting for connection clean up");
}
await pthis.AwaitConnectionReset();
if (LogAdapter.IsWarningEnabled)
{
LogAdapter.LogWarn(LogSource, "Connection is ready");
}
pthis.FireWillRecover();
pthis.ResetConnection();
var didConnect = await pthis.CycleReconnect().ConfigureAwait(false);
if (!didConnect)
{
// Cancelled
pthis.FireRecoveryFailed(new Exception("Could not connect to any host"));
return;
}
if (LogAdapter.IsWarningEnabled)
{
LogAdapter.LogWarn(LogSource, "Reconnected");
}
await pthis.Recover().ConfigureAwait(false);
pthis.FireRecoveryCompleted();
if (LogAdapter.IsWarningEnabled)
{
LogAdapter.LogWarn(LogSource, "Completed");
}
}
catch (Exception ex)
{
if (LogAdapter.IsWarningEnabled)
{
LogAdapter.LogWarn(LogSource, "TryInitiateRecovery error", ex);
}
pthis.HandleRecoveryFatalError(ex);
pthis.FireRecoveryFailed(ex);
}
finally
{
Interlocked.Exchange(ref pthis._inRecovery, 0);
}
}, "RecoveryProc", this);
}
else
{
if (LogAdapter.IsDebugEnabled)
{
LogAdapter.LogDebug(LogSource, "TryInitiateRecovery: recovery in progress. skipping");
}
}
}