public override string ToString() => Format.ToString(EndPoint);
public override string ToString() => ConnectionType + "/" + Format.ToString(ServerEndPoint.EndPoint);
internal static Exception Timeout(ConnectionMultiplexer multiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null) { List <Tuple <string, string> > data = new List <Tuple <string, string> > { Tuple.Create("Message", message.CommandAndKey) }; var sb = new StringBuilder(); if (!string.IsNullOrEmpty(baseErrorMessage)) { sb.Append(baseErrorMessage); if (message != null) { sb.Append(", command=").Append(message.Command); // no key here, note } } else { sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(multiplexer.TimeoutMilliseconds)).Append("ms)"); } // Add timeout data, if we have it if (result == WriteResult.TimeoutBeforeWrite) { Add(data, sb, "Timeout", "timeout", Format.ToString(multiplexer.TimeoutMilliseconds)); try { #if DEBUG if (message.QueuePosition >= 0) { Add(data, sb, "QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue } if ((int)message.ConnectionWriteState >= 0) { Add(data, sb, "WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue } #endif if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta)) { Add(data, sb, "Write-State", null, ws.ToString()); Add(data, sb, "Read-State", null, rs.ToString()); // these might not always be available if (sentDelta >= 0) { Add(data, sb, "OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB"); } if (receivedDelta >= 0) { Add(data, sb, "InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB"); } } } catch { } } AddCommonDetail(data, sb, message, multiplexer, server); sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); sb.Append(timeoutHelpLink); sb.Append(")"); var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = timeoutHelpLink }; CopyDataToException(data, ex); if (multiplexer.IncludeDetailInExceptions) { AddExceptionDetail(ex, message, server, null); } return(ex); }
public PhysicalConnection(PhysicalBridge bridge) { lastWriteTickCount = lastReadTickCount = Environment.TickCount; lastBeatTickCount = 0; connectionType = bridge.ConnectionType; Multiplexer = bridge.Multiplexer; ChannelPrefix = Multiplexer.RawConfig.ChannelPrefix; if (ChannelPrefix?.Length == 0) { ChannelPrefix = null; // null tests are easier than null+empty } var endpoint = bridge.ServerEndPoint.EndPoint; physicalName = connectionType + "#" + Interlocked.Increment(ref totalCount) + "@" + Format.ToString(endpoint); Bridge = bridge; OnCreateEcho(); }
private void MatchResult(RawResult result) { // check to see if it could be an out-of-band pubsub message if (connectionType == ConnectionType.Subscription && result.Type == ResultType.MultiBulk) { // out of band message does not match to a queued message var items = result.GetItems(); if (items.Length >= 3 && items[0].IsEqual(message)) { // special-case the configuration change broadcasts (we don't keep that in the usual pub/sub registry) var configChanged = Multiplexer.ConfigurationChangedChannel; if (configChanged != null && items[1].IsEqual(configChanged)) { EndPoint blame = null; try { if (!items[2].IsEqual(RedisLiterals.ByteWildcard)) { blame = Format.TryParseEndPoint(items[2].GetString()); } } catch { /* no biggie */ } Multiplexer.Trace("Configuration changed: " + Format.ToString(blame), physicalName); Multiplexer.ReconfigureIfNeeded(blame, true, "broadcast"); } // invoke the handlers var channel = items[1].AsRedisChannel(ChannelPrefix, RedisChannel.PatternMode.Literal); Multiplexer.Trace("MESSAGE: " + channel, physicalName); if (!channel.IsNull) { Multiplexer.OnMessage(channel, channel, items[2].AsRedisValue()); } return; // AND STOP PROCESSING! } else if (items.Length >= 4 && items[0].IsEqual(pmessage)) { var channel = items[2].AsRedisChannel(ChannelPrefix, RedisChannel.PatternMode.Literal); Multiplexer.Trace("PMESSAGE: " + channel, physicalName); if (!channel.IsNull) { var sub = items[1].AsRedisChannel(ChannelPrefix, RedisChannel.PatternMode.Pattern); Multiplexer.OnMessage(sub, channel, items[3].AsRedisValue()); } return; // AND STOP PROCESSING! } // if it didn't look like "[p]message", then we still need to process the pending queue } Multiplexer.Trace("Matching result...", physicalName); Message msg; lock (outstanding) { Multiplexer.Trace(outstanding.Count == 0, "Nothing to respond to!", physicalName); msg = outstanding.Dequeue(); } Multiplexer.Trace("Response to: " + msg, physicalName); if (msg.ComputeResult(this, result)) { Bridge.CompleteSyncOrAsync(msg); } }
public void RecordConnectionFailed(ConnectionFailureType failureType, ref SocketManager.ManagerState managerState, Exception innerException = null, [CallerMemberName] string origin = null) { IdentifyFailureType(innerException, ref failureType); managerState = SocketManager.ManagerState.RecordConnectionFailed_OnInternalError; if (failureType == ConnectionFailureType.InternalFailure) { OnInternalError(innerException, origin); } // stop anything new coming in... Bridge.Trace("Failed: " + failureType); int @in = -1, ar = -1; managerState = SocketManager.ManagerState.RecordConnectionFailed_OnDisconnected; Bridge.OnDisconnected(failureType, this, out bool isCurrent, out PhysicalBridge.State oldState); if (oldState == PhysicalBridge.State.ConnectedEstablished) { try { @in = GetAvailableInboundBytes(out ar); } catch { /* best effort only */ } } if (isCurrent && Interlocked.CompareExchange(ref failureReported, 1, 0) == 0) { managerState = SocketManager.ManagerState.RecordConnectionFailed_ReportFailure; int now = Environment.TickCount, lastRead = Thread.VolatileRead(ref lastReadTickCount), lastWrite = Thread.VolatileRead(ref lastWriteTickCount), lastBeat = Thread.VolatileRead(ref lastBeatTickCount); int unansweredRead = Thread.VolatileRead(ref firstUnansweredWriteTickCount); var exMessage = new StringBuilder(failureType.ToString()); var data = new List <Tuple <string, string> >(); if (Multiplexer.IncludeDetailInExceptions) { exMessage.Append(" on " + Format.ToString(Bridge.ServerEndPoint.EndPoint) + "/" + connectionType); data.Add(Tuple.Create("FailureType", failureType.ToString())); data.Add(Tuple.Create("EndPoint", Format.ToString(Bridge.ServerEndPoint.EndPoint))); void add(string lk, string sk, string v) { data.Add(Tuple.Create(lk, v)); exMessage.Append(", ").Append(sk).Append(": ").Append(v); } add("Origin", "origin", origin); add("Input-Buffer", "input-buffer", ioBufferBytes.ToString()); add("Outstanding-Responses", "outstanding", GetSentAwaitingResponseCount().ToString()); add("Last-Read", "last-read", (unchecked (now - lastRead) / 1000) + "s ago"); add("Last-Write", "last-write", (unchecked (now - lastWrite) / 1000) + "s ago"); add("Unanswered-Write", "unanswered-write", (unchecked (now - unansweredRead) / 1000) + "s ago"); add("Keep-Alive", "keep-alive", Bridge.ServerEndPoint.WriteEverySeconds + "s"); add("Pending", "pending", Bridge.GetPendingCount().ToString()); add("Previous-Physical-State", "state", oldState.ToString()); if (@in >= 0) { add("Inbound-Bytes", "in", @in.ToString()); add("Active-Readers", "ar", ar.ToString()); } add("Last-Heartbeat", "last-heartbeat", (lastBeat == 0 ? "never" : ((unchecked (now - lastBeat) / 1000) + "s ago")) + (Bridge.IsBeating ? " (mid-beat)" : "")); add("Last-Multiplexer-Heartbeat", "last-mbeat", Multiplexer.LastHeartbeatSecondsAgo + "s ago"); add("Last-Global-Heartbeat", "global", ConnectionMultiplexer.LastGlobalHeartbeatSecondsAgo + "s ago"); #if FEATURE_SOCKET_MODE_POLL var mgr = Bridge.Multiplexer.SocketManager; add("SocketManager-State", "mgr", mgr.State.ToString()); add("Last-Error", "err", mgr.LastErrorTimeRelative()); #endif } var ex = innerException == null ? new RedisConnectionException(failureType, exMessage.ToString()) : new RedisConnectionException(failureType, exMessage.ToString(), innerException); foreach (var kv in data) { ex.Data["Redis-" + kv.Item1] = kv.Item2; } managerState = SocketManager.ManagerState.RecordConnectionFailed_OnConnectionFailed; Bridge.OnConnectionFailed(this, failureType, ex); } // cleanup managerState = SocketManager.ManagerState.RecordConnectionFailed_FailOutstanding; lock (outstanding) { Bridge.Trace(outstanding.Count != 0, "Failing outstanding messages: " + outstanding.Count); while (outstanding.Count != 0) { var next = outstanding.Dequeue(); Bridge.Trace("Failing: " + next); next.Fail(failureType, innerException); Bridge.CompleteSyncOrAsync(next); } } // burn the socket managerState = SocketManager.ManagerState.RecordConnectionFailed_ShutdownSocket; Multiplexer.SocketManager?.Shutdown(socketToken); }
private static void WriteRaw(Stream stream, long value, bool withLengthPrefix = false) { if (value >= 0 && value <= 9) { if (withLengthPrefix) { stream.WriteByte((byte)'1'); stream.Write(Crlf, 0, 2); } stream.WriteByte((byte)((int)'0' + (int)value)); } else if (value >= 10 && value < 100) { if (withLengthPrefix) { stream.WriteByte((byte)'2'); stream.Write(Crlf, 0, 2); } stream.WriteByte((byte)((int)'0' + ((int)value / 10))); stream.WriteByte((byte)((int)'0' + ((int)value % 10))); } else if (value >= 100 && value < 1000) { int v = (int)value; int units = v % 10; v /= 10; int tens = v % 10, hundreds = v / 10; if (withLengthPrefix) { stream.WriteByte((byte)'3'); stream.Write(Crlf, 0, 2); } stream.WriteByte((byte)((int)'0' + hundreds)); stream.WriteByte((byte)((int)'0' + tens)); stream.WriteByte((byte)((int)'0' + units)); } else if (value < 0 && value >= -9) { if (withLengthPrefix) { stream.WriteByte((byte)'2'); stream.Write(Crlf, 0, 2); } stream.WriteByte((byte)'-'); stream.WriteByte((byte)((int)'0' - (int)value)); } else if (value <= -10 && value > -100) { if (withLengthPrefix) { stream.WriteByte((byte)'3'); stream.Write(Crlf, 0, 2); } value = -value; stream.WriteByte((byte)'-'); stream.WriteByte((byte)((int)'0' + ((int)value / 10))); stream.WriteByte((byte)((int)'0' + ((int)value % 10))); } else { var bytes = Encoding.ASCII.GetBytes(Format.ToString(value)); if (withLengthPrefix) { WriteRaw(stream, bytes.Length, false); } stream.Write(bytes, 0, bytes.Length); } stream.Write(Crlf, 0, 2); }
public void RecordConnectionFailed(ConnectionFailureType failureType, Exception innerException = null, [CallerMemberName] string origin = null) { IdentifyFailureType(innerException, ref failureType); if (failureType == ConnectionFailureType.InternalFailure) { OnInternalError(innerException, origin); } // stop anything new coming in... bridge.Trace("Failed: " + failureType); bool isCurrent; PhysicalBridge.State oldState; bridge.OnDisconnected(failureType, this, out isCurrent, out oldState); if (isCurrent && Interlocked.CompareExchange(ref failureReported, 1, 0) == 0) { //try //{ int now = Environment.TickCount, lastRead = Thread.VolatileRead(ref lastReadTickCount), lastWrite = Thread.VolatileRead(ref lastWriteTickCount), lastBeat = Thread.VolatileRead(ref lastBeatTickCount); string message = failureType + " on " + Format.ToString(bridge.ServerEndPoint.EndPoint) + "/" + connectionType + ", input-buffer: " + ioBufferBytes + ", outstanding: " + GetSentAwaitingResponseCount() + ", last-read: " + unchecked (now - lastRead) / 1000 + "s ago, last-write: " + unchecked (now - lastWrite) / 1000 + "s ago, keep-alive: " + bridge.ServerEndPoint.WriteEverySeconds + "s, pending: " + bridge.GetPendingCount() + ", state: " + oldState + ", last-heartbeat: " + (lastBeat == 0 ? "never" : (unchecked (now - lastBeat) / 1000 + "s ago")) + (bridge.IsBeating ? " (mid-beat)" : "") + ", last-mbeat: " + multiplexer.LastHeartbeatSecondsAgo + "s ago, global: " + ConnectionMultiplexer.LastGlobalHeartbeatSecondsAgo + "s ago"; var ex = innerException == null ? new RedisConnectionException(failureType, message) : new RedisConnectionException(failureType, message, innerException); bridge.OnConnectionFailed(this, failureType, ex); // throw ex; //} //catch (Exception caught) //{ // bridge.OnConnectionFailed(this, failureType, caught); //} } // cleanup lock (outstanding) { bridge.Trace(outstanding.Count != 0, "Failing outstanding messages: " + outstanding.Count); while (outstanding.Count != 0) { var next = outstanding.Dequeue(); bridge.Trace("Failing: " + next); next.Fail(failureType, innerException); bridge.CompleteSyncOrAsync(next); } } // burn the socket var socketManager = multiplexer.SocketManager; if (socketManager != null) { socketManager.Shutdown(socketToken); } }
public override string ToString() { return(Format.ToString(EndPoint)); }
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server) { List <Tuple <string, string> > data = new List <Tuple <string, string> > { Tuple.Create("Message", message.CommandAndKey) }; var sb = new StringBuilder(); if (!string.IsNullOrEmpty(baseErrorMessage)) { sb.Append(baseErrorMessage); } else { sb.Append("Timeout performing ").Append(message.CommandAndKey).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)"); } void add(string lk, string sk, string v) { if (v != null) { data.Add(Tuple.Create(lk, v)); sb.Append(", ").Append(sk).Append(": ").Append(v); } } if (server != null) { server.GetOutstandingCount(message.Command, out int inst, out int qs, out int @in); add("Instantaneous", "inst", inst.ToString()); add("Queue-Awaiting-Response", "qs", qs.ToString()); add("Inbound-Bytes", "in", @in.ToString()); if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0) { var log = server.GetStormLog(message.Command); if (string.IsNullOrWhiteSpace(log)) { Interlocked.Exchange(ref mutiplexer.haveStormLog, 0); } else { Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log); } } add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString()); } add("Manager", "mgr", mutiplexer.SocketManager?.GetState()); add("Client-Name", "clientName", mutiplexer.ClientName); var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy); // only add keyslot if its a valid cluster key slot if (hashSlot != ServerSelectionStrategy.NoSlot) { add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString()); } int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker); add("ThreadPool-IO-Completion", "IOCP", iocp); add("ThreadPool-Workers", "WORKER", worker); data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString())); if (mutiplexer.IncludePerformanceCountersInExceptions) { add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); } sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); sb.Append(timeoutHelpLink); sb.Append(")"); var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = timeoutHelpLink }; if (data != null) { foreach (var kv in data) { ex.Data["Redis-" + kv.Item1] = kv.Item2; } } if (mutiplexer.IncludeDetailInExceptions) { AddDetail(ex, message, server, null); } return(ex); }
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null) { List <Tuple <string, string> > data = new List <Tuple <string, string> > { Tuple.Create("Message", message.CommandAndKey) }; var sb = new StringBuilder(); if (!string.IsNullOrEmpty(baseErrorMessage)) { sb.Append(baseErrorMessage); if (message != null) { sb.Append(", command=").Append(message.Command); // no key here, note } } else { sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)"); } void add(string lk, string sk, string v) { if (v != null) { if (lk != null) { data.Add(Tuple.Create(lk, v)); } if (sk != null) { sb.Append(", ").Append(sk).Append(": ").Append(v); } } } // Add timeout data, if we have it if (result == WriteResult.TimeoutBeforeWrite) { add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds)); try { #if DEBUG if (message.QueuePosition >= 0) { add("QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue } if ((int)message.ConnectionWriteState >= 0) { add("WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue } #endif if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta)) { add("Write-State", null, ws.ToString()); add("Read-State", null, rs.ToString()); // these might not always be available if (sentDelta >= 0) { add("OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB"); } if (receivedDelta >= 0) { add("InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB"); } } } catch { } } if (message != null) { message.TryGetHeadMessages(out var now, out var next); if (now != null) { add("Message-Current", "active", mutiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString()); } if (next != null) { add("Message-Next", "next", mutiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString()); } } // Add server data, if we have it if (server != null) { server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite, out var bs, out var rs, out var ws); switch (rs) { case PhysicalConnection.ReadStatus.CompletePendingMessageAsync: case PhysicalConnection.ReadStatus.CompletePendingMessageSync: sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** "); break; } add("OpsSinceLastHeartbeat", "inst", inst.ToString()); add("Queue-Awaiting-Write", "qu", qu.ToString()); add("Queue-Awaiting-Response", "qs", qs.ToString()); add("Active-Writer", "aw", aw.ToString()); if (qu != 0) { add("Backlog-Writer", "bw", bs.ToString()); } if (rs != PhysicalConnection.ReadStatus.NA) { add("Read-State", "rs", rs.ToString()); } if (ws != PhysicalConnection.WriteStatus.NA) { add("Write-State", "ws", ws.ToString()); } if (@in >= 0) { add("Inbound-Bytes", "in", @in.ToString()); } if (toRead >= 0) { add("Inbound-Pipe-Bytes", "in-pipe", toRead.ToString()); } if (toWrite >= 0) { add("Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString()); } if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0) { var log = server.GetStormLog(message.Command); if (string.IsNullOrWhiteSpace(log)) { Interlocked.Exchange(ref mutiplexer.haveStormLog, 0); } else { Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log); } } add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/", "")); } add("Manager", "mgr", mutiplexer.SocketManager?.GetState()); add("Client-Name", "clientName", mutiplexer.ClientName); var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy); // only add keyslot if its a valid cluster key slot if (hashSlot != ServerSelectionStrategy.NoSlot) { add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString()); } int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker); add("ThreadPool-IO-Completion", "IOCP", iocp); add("ThreadPool-Workers", "WORKER", worker); data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString())); if (mutiplexer.IncludePerformanceCountersInExceptions) { add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); } add("Version", "v", GetLibVersion()); sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); sb.Append(timeoutHelpLink); sb.Append(")"); var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = timeoutHelpLink }; if (data != null) { var exData = ex.Data; foreach (var kv in data) { exData["Redis-" + kv.Item1] = kv.Item2; } } if (mutiplexer.IncludeDetailInExceptions) { AddDetail(ex, message, server, null); } return(ex); }
public override string ToString() { return(connectionType + "/" + Format.ToString(serverEndPoint.EndPoint)); }