public void CheckForStaleConnection(ref SocketManager.ManagerState managerState) { int firstUnansweredWrite = VolatileWrapper.Read(ref firstUnansweredWriteTickCount); DebugEmulateStaleConnection(ref firstUnansweredWrite); int now = Environment.TickCount; if (firstUnansweredWrite != 0 && (now - firstUnansweredWrite) > this.Multiplexer.RawConfig.ResponseTimeout) { this.RecordConnectionFailed(ConnectionFailureType.SocketFailure, ref managerState, origin: "CheckForStaleConnection"); } }
public void RecordConnectionFailed(ConnectionFailureType failureType, ref SocketManager.ManagerState managerState, Exception innerException = null, [CallerMemberName] string origin = null) { IdentifyFailureType(innerException, ref failureType); managerState = SocketManager.ManagerState.RecordConnectionFailed_OnInternalError; if (failureType == ConnectionFailureType.InternalFailure) { OnInternalError(innerException, origin); } // stop anything new coming in... Bridge.Trace("Failed: " + failureType); bool isCurrent; PhysicalBridge.State oldState; int @in = -1, ar = -1; managerState = SocketManager.ManagerState.RecordConnectionFailed_OnDisconnected; Bridge.OnDisconnected(failureType, this, out isCurrent, out oldState); if (oldState == PhysicalBridge.State.ConnectedEstablished) { try { @in = GetAvailableInboundBytes(out ar); } catch { /* best effort only */ } } if (isCurrent && Interlocked.CompareExchange(ref failureReported, 1, 0) == 0) { managerState = SocketManager.ManagerState.RecordConnectionFailed_ReportFailure; int now = Environment.TickCount, lastRead = VolatileWrapper.Read(ref lastReadTickCount), lastWrite = VolatileWrapper.Read(ref lastWriteTickCount), lastBeat = VolatileWrapper.Read(ref lastBeatTickCount); int unansweredRead = VolatileWrapper.Read(ref firstUnansweredWriteTickCount); var exMessage = new StringBuilder(failureType + " on " + Format.ToString(Bridge.ServerEndPoint.EndPoint) + "/" + connectionType); var data = new List <Tuple <string, string> > { Tuple.Create("FailureType", failureType.ToString()), Tuple.Create("EndPoint", Format.ToString(Bridge.ServerEndPoint.EndPoint)) }; Action <string, string, string> add = (lk, sk, v) => { data.Add(Tuple.Create(lk, v)); exMessage.Append(", " + sk + ": " + v); }; add("Origin", "origin", origin); add("Input-Buffer", "input-buffer", ioBufferBytes.ToString()); add("Outstanding-Responses", "outstanding", GetSentAwaitingResponseCount().ToString()); add("Last-Read", "last-read", unchecked (now - lastRead) / 1000 + "s ago"); add("Last-Write", "last-write", unchecked (now - lastWrite) / 1000 + "s ago"); add("Unanswered-Write", "unanswered-write", unchecked (now - unansweredRead) / 1000 + "s ago"); add("Keep-Alive", "keep-alive", Bridge.ServerEndPoint.WriteEverySeconds + "s"); add("Pending", "pending", Bridge.GetPendingCount().ToString()); add("Previous-Physical-State", "state", oldState.ToString()); if (@in >= 0) { add("Inbound-Bytes", "in", @in.ToString()); add("Active-Readers", "ar", ar.ToString()); } add("Last-Heartbeat", "last-heartbeat", (lastBeat == 0 ? "never" : (unchecked (now - lastBeat) / 1000 + "s ago")) + (Bridge.IsBeating ? " (mid-beat)" : "")); add("Last-Multiplexer-Heartbeat", "last-mbeat", Multiplexer.LastHeartbeatSecondsAgo + "s ago"); add("Last-Global-Heartbeat", "global", ConnectionMultiplexer.LastGlobalHeartbeatSecondsAgo + "s ago"); #if FEATURE_SOCKET_MODE_POLL var mgr = Bridge.Multiplexer.SocketManager; add("SocketManager-State", "mgr", mgr.State.ToString()); add("Last-Error", "err", mgr.LastErrorTimeRelative()); #endif var ex = innerException == null ? new RedisConnectionException(failureType, exMessage.ToString()) : new RedisConnectionException(failureType, exMessage.ToString(), innerException); foreach (var kv in data) { ex.Data["Redis-" + kv.Item1] = kv.Item2; } managerState = SocketManager.ManagerState.RecordConnectionFailed_OnConnectionFailed; Bridge.OnConnectionFailed(this, failureType, ex); } // cleanup managerState = SocketManager.ManagerState.RecordConnectionFailed_FailOutstanding; lock (outstanding) { Bridge.Trace(outstanding.Count != 0, "Failing outstanding messages: " + outstanding.Count); while (outstanding.Count != 0) { var next = outstanding.Dequeue(); Bridge.Trace("Failing: " + next); next.Fail(failureType, innerException); Bridge.CompleteSyncOrAsync(next); } } // burn the socket managerState = SocketManager.ManagerState.RecordConnectionFailed_ShutdownSocket; Multiplexer.SocketManager?.Shutdown(socketToken); }
internal void OnHeartbeat(bool ifConnectedOnly) { bool runThisTime = false; try { runThisTime = !isDisposed && Interlocked.CompareExchange(ref beating, 1, 0) == 0; if (!runThisTime) { return; } uint index = (uint)Interlocked.Increment(ref profileLogIndex); long newSampleCount = Interlocked.Read(ref operationCount); Interlocked.Exchange(ref profileLog[index % ProfileLogSamples], newSampleCount); Interlocked.Exchange(ref profileLastLog, newSampleCount); Trace("OnHeartbeat: " + (State)state); switch (state) { case (int)State.Connecting: int connectTimeMilliseconds = unchecked (Environment.TickCount - VolatileWrapper.Read(ref connectStartTicks)); if (connectTimeMilliseconds >= Multiplexer.RawConfig.ConnectTimeout) { LastException = ExceptionFactory.UnableToConnect("ConnectTimeout"); Trace("Aborting connect"); // abort and reconnect var snapshot = physical; bool isCurrent; State oldState; OnDisconnected(ConnectionFailureType.UnableToConnect, snapshot, out isCurrent, out oldState); using (snapshot) { } // dispose etc TryConnect(null); } if (!ifConnectedOnly) { AbortUnsent(); } break; case (int)State.ConnectedEstablishing: case (int)State.ConnectedEstablished: var tmp = physical; if (tmp != null) { if (state == (int)State.ConnectedEstablished) { tmp.Bridge.ServerEndPoint.ClearUnselectable(UnselectableFlags.DidNotRespond); } tmp.OnHeartbeat(); int writeEverySeconds = ServerEndPoint.WriteEverySeconds, checkConfigSeconds = Multiplexer.RawConfig.ConfigCheckSeconds; if (state == (int)State.ConnectedEstablished && ConnectionType == ConnectionType.Interactive && checkConfigSeconds > 0 && ServerEndPoint.LastInfoReplicationCheckSecondsAgo >= checkConfigSeconds && ServerEndPoint.CheckInfoReplication()) { // that serves as a keep-alive, if it is accepted } else if (writeEverySeconds > 0 && tmp.LastWriteSecondsAgo >= writeEverySeconds) { Trace("OnHeartbeat - overdue"); if (state == (int)State.ConnectedEstablished) { KeepAlive(); } else { bool ignore; State oldState; OnDisconnected(ConnectionFailureType.SocketFailure, tmp, out ignore, out oldState); } } else if (!queue.Any() && tmp.GetSentAwaitingResponseCount() != 0) { // there's a chance this is a dead socket; sending data will shake that // up a bit, so if we have an empty unsent queue and a non-empty sent // queue, test the socket KeepAlive(); } } break; case (int)State.Disconnected: if (!ifConnectedOnly) { AbortUnsent(); Multiplexer.Trace("Resurrecting " + this.ToString()); GetConnection(null); } break; default: if (!ifConnectedOnly) { AbortUnsent(); } break; } } catch (Exception ex) { OnInternalError(ex); Trace("OnHeartbeat error: " + ex.Message); } finally { if (runThisTime) { Interlocked.Exchange(ref beating, 0); } } }