protected void ReconnectNow(INode node)
{
try
{
if (shutdownToken.IsCancellationRequested)
{
return;
}
node.Connect(shutdownToken.Token);
ReAddNode(node);
ioQueue.Add(node); // trigger IO on this node
}
catch (OperationCanceledException)
{
LogTo.Info("Cluster was shut down during reconnect, aborting.");
}
catch (Exception e)
{
LogTo.Error(e, "Failed to reconnect");
ScheduleReconnect(node);
}
}
/// <summary>
/// Mark the specified node as working.
/// </summary>
/// <param name="node"></param>
/// <remarks>Can be called from a background thread (Task pool)</remarks>
protected void ReAddNode(INode node)
{
LogTo.Info($"Node {node} was reconnected");
// serialize the reconnect attempts to make
// IReconnectPolicy and INodeLocator implementations simpler
lock (ReconnectLock)
{
reconnectPolicy.Reset(node);
var original = Volatile.Read(ref workingNodes);
// even though we're locking we still do the CAS,
// because the IO thread does not use any locking
while (true)
{
// append the new node to the list of working nodes
var updated = new INode[original.Length + 1];
Array.Copy(original, 0, updated, 0, original.Length);
updated[original.Length] = node;
var previous = Interlocked.CompareExchange(ref workingNodes, updated, original);
if (ReferenceEquals(original, previous))
{
locator.Initialize(updated);
break;
}
original = previous;
}
}
}
public virtual void Connect(CancellationToken token)
{
Debug.Assert(currentWriteCopier == null);
Debug.Assert(inprogressResponse == null);
Debug.Assert(readQueue.Count == 0);
LogTo.Info($"Connecting node to {name}");
socket.Connect(endpoint, token);
mustReconnect = false;
IsAlive = true;
}
private bool FailMe(Exception e)
{
lock (failLock)
{
LogTo.Error(e, $"Node {name} has failed during IO.");
var fail = (e is IOException) ? e : new IOException("io fail; see inner exception", e);
// empty all queues
FailQueue(writeQueue, fail);
FailQueue(readQueue, fail);
npm.ResetQueues();
// kill the partially sent op (if any)
if (currentWriteCopier != null)
{
currentWriteOp.Task.SetException(fail);
currentWriteOp = OpQueueEntry.Empty;
currentWriteCopier = null;
}
// kill the partially read response
if (inprogressResponse != null)
{
inprogressResponse.Dispose();
inprogressResponse = null;
}
_FinishedReading();
_FinishedWriting();
// mark as dead if policy says so...
if (failurePolicy.ShouldFail(this))
{
IsAlive = false;
return(true);
}
// ...otherwise reconnect immediately (when it's our turn)
mustReconnect = true;
LogTo.Info($"Node {endpoint} will reconnect immediately.");
// reconnect from IO thread
owner.NeedsIO(this);
return(false);
}
}
private void PerformConnect(IPEndPoint endpoint, CancellationToken token)
{
this.endpoint = endpoint;
this.name = endpoint.ToString();
this.IsAlive = false;
var sw = Stopwatch.StartNew();
InitBuffers();
using (var mre = new ManualResetEventSlim(false))
using (var opt = new SocketAsyncEventArgs {
RemoteEndPoint = endpoint
})
{
CoreEventSource.ConnectStart(name);
opt.Completed += (a, b) => mre.Set();
RecreateSocket();
try
{
if (socket.ConnectAsync(opt) &&
!mre.Wait((int)ConnectionTimeout.TotalMilliseconds, token))
{
CoreEventSource.ConnectFail(name, SocketError.TimedOut);
Socket.CancelConnectAsync(opt);
throw new TimeoutException($"Connection timeout {ConnectionTimeout} has been exceeded while trying to connect to {endpoint}");
}
if (opt.SocketError != SocketError.Success)
{
CoreEventSource.ConnectFail(name, opt.SocketError);
throw new IOException($"Could not connect to {endpoint}");
}
IsAlive = true;
}
finally
{
LogTo.Info($"Connected to {endpoint} in {sw.ElapsedMilliseconds} msec");
}
}
}
/// <summary>
/// Schedules a failed node for reconnection.
/// </summary>
/// <param name="node"></param>
protected virtual void ScheduleReconnect(INode node)
{
LogTo.Info($"Scheduling reconnect for {node}");
var when = reconnectPolicy.Schedule(node);
if (when == TimeSpan.Zero)
{
LogTo.Info("Will reconnect now");
ReconnectNow(node);
}
else
{
LogTo.Info($"Will reconnect after {when}");
Task
.Delay(when, shutdownToken.Token)
.ContinueWith(_ => ReconnectNow(node), TaskContinuationOptions.OnlyOnRanToCompletion);
}
}
private void DestroySocket()
{
if (socket != null)
{
try
{
using (socket)
{
socket.Shutdown(SocketShutdown.Both);
socket.Close();
}
}
catch (Exception e)
{
LogTo.Info(e, "Exception while destroying socket.");
}
socket = null;
}
}
