internal static Exception NoConnectionAvailable(bool includeDetail, bool includePerformanceCounters, RedisCommand command, Message message, ServerEndPoint server, ReadOnlySpan <ServerEndPoint> serverSnapshot) { string commandLabel = GetLabel(includeDetail, command, message); if (server != null) { //if we already have the serverEndpoint for connection failure use that //otherwise it would output state of all the endpoints serverSnapshot = new ServerEndPoint[] { server }; } var innerException = PopulateInnerExceptions(serverSnapshot); StringBuilder exceptionmessage = new StringBuilder("No connection is available to service this operation: ").Append(commandLabel); string innermostExceptionstring = GetInnerMostExceptionMessage(innerException); if (!string.IsNullOrEmpty(innermostExceptionstring)) { exceptionmessage.Append("; ").Append(innermostExceptionstring); } if (includeDetail) { exceptionmessage.Append("; ").Append(PerfCounterHelper.GetThreadPoolAndCPUSummary(includePerformanceCounters)); } var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, exceptionmessage.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); if (includeDetail) { AddDetail(ex, message, server, commandLabel); } return(ex); }
private static void AddCommonDetail( List <Tuple <string, string> > data, StringBuilder sb, Message message, ConnectionMultiplexer multiplexer, ServerEndPoint server ) { if (message != null) { message.TryGetHeadMessages(out var now, out var next); if (now != null) { Add(data, sb, "Message-Current", "active", multiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString()); } if (next != null) { Add(data, sb, "Message-Next", "next", multiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString()); } } // Add server data, if we have it if (server != null && message != null) { var bs = server.GetBridgeStatus(message.IsForSubscriptionBridge ? ConnectionType.Subscription: ConnectionType.Interactive); switch (bs.Connection.ReadStatus) { case PhysicalConnection.ReadStatus.CompletePendingMessageAsync: case PhysicalConnection.ReadStatus.CompletePendingMessageSync: sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** "); break; } Add(data, sb, "OpsSinceLastHeartbeat", "inst", bs.MessagesSinceLastHeartbeat.ToString()); Add(data, sb, "Queue-Awaiting-Write", "qu", bs.BacklogMessagesPending.ToString()); Add(data, sb, "Queue-Awaiting-Response", "qs", bs.Connection.MessagesSentAwaitingResponse.ToString()); Add(data, sb, "Active-Writer", "aw", bs.IsWriterActive.ToString()); if (bs.BacklogMessagesPending != 0) { Add(data, sb, "Backlog-Writer", "bw", bs.BacklogStatus.ToString()); } if (bs.Connection.ReadStatus != PhysicalConnection.ReadStatus.NA) { Add(data, sb, "Read-State", "rs", bs.Connection.ReadStatus.ToString()); } if (bs.Connection.WriteStatus != PhysicalConnection.WriteStatus.NA) { Add(data, sb, "Write-State", "ws", bs.Connection.WriteStatus.ToString()); } if (bs.Connection.BytesAvailableOnSocket >= 0) { Add(data, sb, "Inbound-Bytes", "in", bs.Connection.BytesAvailableOnSocket.ToString()); } if (bs.Connection.BytesInReadPipe >= 0) { Add(data, sb, "Inbound-Pipe-Bytes", "in-pipe", bs.Connection.BytesInReadPipe.ToString()); } if (bs.Connection.BytesInWritePipe >= 0) { Add(data, sb, "Outbound-Pipe-Bytes", "out-pipe", bs.Connection.BytesInWritePipe.ToString()); } if (multiplexer.StormLogThreshold >= 0 && bs.Connection.MessagesSentAwaitingResponse >= multiplexer.StormLogThreshold && Interlocked.CompareExchange(ref multiplexer.haveStormLog, 1, 0) == 0) { var log = server.GetStormLog(message); if (string.IsNullOrWhiteSpace(log)) { Interlocked.Exchange(ref multiplexer.haveStormLog, 0); } else { Interlocked.Exchange(ref multiplexer.stormLogSnapshot, log); } } Add(data, sb, "Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/", "")); } Add(data, sb, "Multiplexer-Connects", "mc", $"{multiplexer._connectAttemptCount}/{multiplexer._connectCompletedCount}/{multiplexer._connectionCloseCount}"); Add(data, sb, "Manager", "mgr", multiplexer.SocketManager?.GetState()); Add(data, sb, "Client-Name", "clientName", multiplexer.ClientName); if (message != null) { var hashSlot = message.GetHashSlot(multiplexer.ServerSelectionStrategy); // only add keyslot if its a valid cluster key slot if (hashSlot != ServerSelectionStrategy.NoSlot) { Add(data, sb, "Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(multiplexer.ServerSelectionStrategy).ToString()); } } int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker, out string workItems); Add(data, sb, "ThreadPool-IO-Completion", "IOCP", iocp); Add(data, sb, "ThreadPool-Workers", "WORKER", worker); if (workItems != null) { Add(data, sb, "ThreadPool-Items", "POOL", workItems); } data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString())); if (multiplexer.IncludePerformanceCountersInExceptions) { Add(data, sb, "Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); } Add(data, sb, "Version", "v", GetLibVersion()); }
internal static Exception NoConnectionAvailable( ConnectionMultiplexer multiplexer, Message message, ServerEndPoint server, ReadOnlySpan <ServerEndPoint> serverSnapshot = default, RedisCommand command = default) { string commandLabel = GetLabel(multiplexer.IncludeDetailInExceptions, message?.Command ?? command, message); if (server != null) { //if we already have the serverEndpoint for connection failure use that //otherwise it would output state of all the endpoints serverSnapshot = new ServerEndPoint[] { server }; } var innerException = PopulateInnerExceptions(serverSnapshot == default ? multiplexer.GetServerSnapshot() : serverSnapshot); // Try to get a useful error message for the user. long attempts = multiplexer._connectAttemptCount, completions = multiplexer._connectCompletedCount; string initialMessage; // We only need to customize the connection if we're aborting on connect fail // The "never" case would have thrown, if this was true if (!multiplexer.RawConfig.AbortOnConnectFail && attempts <= multiplexer.RawConfig.ConnectRetry && completions == 0) { // Initial attempt, attempted use before an async connection completes initialMessage = $"Connection to Redis never succeeded (attempts: {attempts} - connection likely in-progress), unable to service operation: "; } else if (!multiplexer.RawConfig.AbortOnConnectFail && attempts > multiplexer.RawConfig.ConnectRetry && completions == 0) { // Attempted use after a full initial retry connect count # of failures // This can happen in Azure often, where user disables abort and has the wrong config initialMessage = $"Connection to Redis never succeeded (attempts: {attempts} - check your config), unable to service operation: "; } else { // Default if we don't have a more useful error message here based on circumstances initialMessage = "No connection is active/available to service this operation: "; } StringBuilder sb = new StringBuilder(initialMessage); sb.Append(commandLabel); string innermostExceptionstring = GetInnerMostExceptionMessage(innerException); if (!string.IsNullOrEmpty(innermostExceptionstring)) { sb.Append("; ").Append(innermostExceptionstring); } // Add counters and exception data if we have it List <Tuple <string, string> > data = null; if (multiplexer.IncludeDetailInExceptions) { data = new List <Tuple <string, string> >(); AddCommonDetail(data, sb, message, multiplexer, server); } var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown); if (multiplexer.IncludeDetailInExceptions) { CopyDataToException(data, ex); sb.Append("; ").Append(PerfCounterHelper.GetThreadPoolAndCPUSummary(multiplexer.IncludePerformanceCountersInExceptions)); AddExceptionDetail(ex, message, server, commandLabel); } return(ex); }
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null) { List <Tuple <string, string> > data = new List <Tuple <string, string> > { Tuple.Create("Message", message.CommandAndKey) }; var sb = new StringBuilder(); if (!string.IsNullOrEmpty(baseErrorMessage)) { sb.Append(baseErrorMessage); } else { sb.Append("Timeout performing ").Append(message.CommandAndKey).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)"); } void add(string lk, string sk, string v) { if (v != null) { data.Add(Tuple.Create(lk, v)); sb.Append(", ").Append(sk).Append(": ").Append(v); } } // Add timeout data, if we have it if (result == WriteResult.TimeoutBeforeWrite) { add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds)); try { if (message.TryGetPhysicalState(out var state, out var sentDelta, out var receivedDelta)) { add("PhysicalState", "phys", state.ToString()); // these might not always be available if (sentDelta >= 0) { add("OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB"); } if (receivedDelta >= 0) { add("InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB"); } } } catch { } } // Add server data, if we have it if (server != null) { server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite); add("OpsSinceLastHeartbeat", "inst", inst.ToString()); add("Queue-Awaiting-Write", "qu", qu.ToString()); add("Queue-Awaiting-Response", "qs", qs.ToString()); add("Active-Writer", "aw", aw.ToString()); if (@in >= 0) { add("Inbound-Bytes", "in", @in.ToString()); } if (toRead >= 0) { add("Inbound-Pipe-Bytes", "in-pipe", toRead.ToString()); } if (toWrite >= 0) { add("Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString()); } if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0) { var log = server.GetStormLog(message.Command); if (string.IsNullOrWhiteSpace(log)) { Interlocked.Exchange(ref mutiplexer.haveStormLog, 0); } else { Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log); } } add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString()); } add("Manager", "mgr", mutiplexer.SocketManager?.GetState()); add("Client-Name", "clientName", mutiplexer.ClientName); var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy); // only add keyslot if its a valid cluster key slot if (hashSlot != ServerSelectionStrategy.NoSlot) { add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString()); } int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker); add("ThreadPool-IO-Completion", "IOCP", iocp); add("ThreadPool-Workers", "WORKER", worker); data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString())); if (mutiplexer.IncludePerformanceCountersInExceptions) { add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); } add("Version", "v", GetLibVersion()); sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); sb.Append(timeoutHelpLink); sb.Append(")"); var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = timeoutHelpLink }; if (data != null) { foreach (var kv in data) { ex.Data["Redis-" + kv.Item1] = kv.Item2; } } if (mutiplexer.IncludeDetailInExceptions) { AddDetail(ex, message, server, null); } return(ex); }
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server) { List <Tuple <string, string> > data = new List <Tuple <string, string> > { Tuple.Create("Message", message.CommandAndKey) }; var sb = new StringBuilder(); if (!string.IsNullOrEmpty(baseErrorMessage)) { sb.Append(baseErrorMessage); } else { sb.Append("Timeout performing ").Append(message.CommandAndKey).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)"); } void add(string lk, string sk, string v) { if (v != null) { data.Add(Tuple.Create(lk, v)); sb.Append(", ").Append(sk).Append(": ").Append(v); } } if (server != null) { server.GetOutstandingCount(message.Command, out int inst, out int qs, out int @in); add("Instantaneous", "inst", inst.ToString()); add("Queue-Awaiting-Response", "qs", qs.ToString()); if (@in >= 0) { add("Inbound-Bytes", "in", @in.ToString()); } if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0) { var log = server.GetStormLog(message.Command); if (string.IsNullOrWhiteSpace(log)) { Interlocked.Exchange(ref mutiplexer.haveStormLog, 0); } else { Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log); } } add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString()); } add("Manager", "mgr", mutiplexer.SocketManager?.GetState()); add("Client-Name", "clientName", mutiplexer.ClientName); var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy); // only add keyslot if its a valid cluster key slot if (hashSlot != ServerSelectionStrategy.NoSlot) { add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString()); } int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker); add("ThreadPool-IO-Completion", "IOCP", iocp); add("ThreadPool-Workers", "WORKER", worker); data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString())); if (mutiplexer.IncludePerformanceCountersInExceptions) { add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); } add("Version", "v", GetLibVersion()); sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); sb.Append(timeoutHelpLink); sb.Append(")"); var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = timeoutHelpLink }; if (data != null) { foreach (var kv in data) { ex.Data["Redis-" + kv.Item1] = kv.Item2; } } if (mutiplexer.IncludeDetailInExceptions) { AddDetail(ex, message, server, null); } return(ex); }
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null) { List <Tuple <string, string> > data = new List <Tuple <string, string> > { Tuple.Create("Message", message.CommandAndKey) }; var sb = new StringBuilder(); if (!string.IsNullOrEmpty(baseErrorMessage)) { sb.Append(baseErrorMessage); if (message != null) { sb.Append(", command=").Append(message.Command); // no key here, note } } else { sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)"); } void add(string lk, string sk, string v) { if (v != null) { if (lk != null) { data.Add(Tuple.Create(lk, v)); } if (sk != null) { sb.Append(", ").Append(sk).Append(": ").Append(v); } } } // Add timeout data, if we have it if (result == WriteResult.TimeoutBeforeWrite) { add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds)); try { #if DEBUG if (message.QueuePosition >= 0) { add("QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue } if ((int)message.ConnectionWriteState >= 0) { add("WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue } #endif if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta)) { add("Write-State", null, ws.ToString()); add("Read-State", null, rs.ToString()); // these might not always be available if (sentDelta >= 0) { add("OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB"); } if (receivedDelta >= 0) { add("InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB"); } } } catch { } } if (message != null) { message.TryGetHeadMessages(out var now, out var next); if (now != null) { add("Message-Current", "active", mutiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString()); } if (next != null) { add("Message-Next", "next", mutiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString()); } } // Add server data, if we have it if (server != null) { server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite, out var bs, out var rs, out var ws); switch (rs) { case PhysicalConnection.ReadStatus.CompletePendingMessageAsync: case PhysicalConnection.ReadStatus.CompletePendingMessageSync: sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** "); break; } add("OpsSinceLastHeartbeat", "inst", inst.ToString()); add("Queue-Awaiting-Write", "qu", qu.ToString()); add("Queue-Awaiting-Response", "qs", qs.ToString()); add("Active-Writer", "aw", aw.ToString()); if (qu != 0) { add("Backlog-Writer", "bw", bs.ToString()); } if (rs != PhysicalConnection.ReadStatus.NA) { add("Read-State", "rs", rs.ToString()); } if (ws != PhysicalConnection.WriteStatus.NA) { add("Write-State", "ws", ws.ToString()); } if (@in >= 0) { add("Inbound-Bytes", "in", @in.ToString()); } if (toRead >= 0) { add("Inbound-Pipe-Bytes", "in-pipe", toRead.ToString()); } if (toWrite >= 0) { add("Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString()); } if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0) { var log = server.GetStormLog(message.Command); if (string.IsNullOrWhiteSpace(log)) { Interlocked.Exchange(ref mutiplexer.haveStormLog, 0); } else { Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log); } } add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString()); } add("Manager", "mgr", mutiplexer.SocketManager?.GetState()); add("Client-Name", "clientName", mutiplexer.ClientName); var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy); // only add keyslot if its a valid cluster key slot if (hashSlot != ServerSelectionStrategy.NoSlot) { add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString()); } int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker); add("ThreadPool-IO-Completion", "IOCP", iocp); add("ThreadPool-Workers", "WORKER", worker); data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString())); if (mutiplexer.IncludePerformanceCountersInExceptions) { add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent()); } add("Version", "v", GetLibVersion()); sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: "); sb.Append(timeoutHelpLink); sb.Append(")"); var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown) { HelpLink = timeoutHelpLink }; if (data != null) { var exData = ex.Data; foreach (var kv in data) { exData["Redis-" + kv.Item1] = kv.Item2; } } if (mutiplexer.IncludeDetailInExceptions) { AddDetail(ex, message, server, null); } return(ex); }