protected void ReconnectNow(INode node) { try { if (shutdownToken.IsCancellationRequested) { return; } node.Connect(shutdownToken.Token); ReAddNode(node); ioQueue.Add(node); // trigger IO on this node } catch (OperationCanceledException) { LogTo.Info("Cluster was shut down during reconnect, aborting."); } catch (Exception e) { LogTo.Error(e, "Failed to reconnect"); ScheduleReconnect(node); } }
/// <summary> /// Mark the specified node as working. /// </summary> /// <param name="node"></param> /// <remarks>Can be called from a background thread (Task pool)</remarks> protected void ReAddNode(INode node) { LogTo.Info($"Node {node} was reconnected"); // serialize the reconnect attempts to make // IReconnectPolicy and INodeLocator implementations simpler lock (ReconnectLock) { reconnectPolicy.Reset(node); var original = Volatile.Read(ref workingNodes); // even though we're locking we still do the CAS, // because the IO thread does not use any locking while (true) { // append the new node to the list of working nodes var updated = new INode[original.Length + 1]; Array.Copy(original, 0, updated, 0, original.Length); updated[original.Length] = node; var previous = Interlocked.CompareExchange(ref workingNodes, updated, original); if (ReferenceEquals(original, previous)) { locator.Initialize(updated); break; } original = previous; } } }
public virtual void Connect(CancellationToken token) { Debug.Assert(currentWriteCopier == null); Debug.Assert(inprogressResponse == null); Debug.Assert(readQueue.Count == 0); LogTo.Info($"Connecting node to {name}"); socket.Connect(endpoint, token); mustReconnect = false; IsAlive = true; }
private bool FailMe(Exception e) { lock (failLock) { LogTo.Error(e, $"Node {name} has failed during IO."); var fail = (e is IOException) ? e : new IOException("io fail; see inner exception", e); // empty all queues FailQueue(writeQueue, fail); FailQueue(readQueue, fail); npm.ResetQueues(); // kill the partially sent op (if any) if (currentWriteCopier != null) { currentWriteOp.Task.SetException(fail); currentWriteOp = OpQueueEntry.Empty; currentWriteCopier = null; } // kill the partially read response if (inprogressResponse != null) { inprogressResponse.Dispose(); inprogressResponse = null; } _FinishedReading(); _FinishedWriting(); // mark as dead if policy says so... if (failurePolicy.ShouldFail(this)) { IsAlive = false; return(true); } // ...otherwise reconnect immediately (when it's our turn) mustReconnect = true; LogTo.Info($"Node {endpoint} will reconnect immediately."); // reconnect from IO thread owner.NeedsIO(this); return(false); } }
private void PerformConnect(IPEndPoint endpoint, CancellationToken token) { this.endpoint = endpoint; this.name = endpoint.ToString(); this.IsAlive = false; var sw = Stopwatch.StartNew(); InitBuffers(); using (var mre = new ManualResetEventSlim(false)) using (var opt = new SocketAsyncEventArgs { RemoteEndPoint = endpoint }) { CoreEventSource.ConnectStart(name); opt.Completed += (a, b) => mre.Set(); RecreateSocket(); try { if (socket.ConnectAsync(opt) && !mre.Wait((int)ConnectionTimeout.TotalMilliseconds, token)) { CoreEventSource.ConnectFail(name, SocketError.TimedOut); Socket.CancelConnectAsync(opt); throw new TimeoutException($"Connection timeout {ConnectionTimeout} has been exceeded while trying to connect to {endpoint}"); } if (opt.SocketError != SocketError.Success) { CoreEventSource.ConnectFail(name, opt.SocketError); throw new IOException($"Could not connect to {endpoint}"); } IsAlive = true; } finally { LogTo.Info($"Connected to {endpoint} in {sw.ElapsedMilliseconds} msec"); } } }
/// <summary> /// Schedules a failed node for reconnection. /// </summary> /// <param name="node"></param> protected virtual void ScheduleReconnect(INode node) { LogTo.Info($"Scheduling reconnect for {node}"); var when = reconnectPolicy.Schedule(node); if (when == TimeSpan.Zero) { LogTo.Info("Will reconnect now"); ReconnectNow(node); } else { LogTo.Info($"Will reconnect after {when}"); Task .Delay(when, shutdownToken.Token) .ContinueWith(_ => ReconnectNow(node), TaskContinuationOptions.OnlyOnRanToCompletion); } }
private void DestroySocket() { if (socket != null) { try { using (socket) { socket.Shutdown(SocketShutdown.Both); socket.Close(); } } catch (Exception e) { LogTo.Info(e, "Exception while destroying socket."); } socket = null; } }