internal static Exception Timeout(bool includeDetail, string errorMessage, Message message, ServerEndPoint server)
{
var ex = new RedisTimeoutException(errorMessage, message?.Status ?? CommandStatus.Unknown);
if (includeDetail)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
internal static Exception Timeout(ConnectionMultiplexer multiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
if (message != null)
{
sb.Append(", command=").Append(message.Command); // no key here, note
}
}
else
{
sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(multiplexer.TimeoutMilliseconds)).Append("ms)");
}
// Add timeout data, if we have it
if (result == WriteResult.TimeoutBeforeWrite)
{
Add(data, sb, "Timeout", "timeout", Format.ToString(multiplexer.TimeoutMilliseconds));
try
{
if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta))
{
Add(data, sb, "Write-State", null, ws.ToString());
Add(data, sb, "Read-State", null, rs.ToString());
// these might not always be available
if (sentDelta >= 0)
{
Add(data, sb, "OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB");
}
if (receivedDelta >= 0)
{
Add(data, sb, "InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB");
}
}
}
catch { }
}
AddCommonDetail(data, sb, message, multiplexer, server);
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(')');
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
CopyDataToException(data, ex);
if (multiplexer.IncludeDetailInExceptions)
{
AddExceptionDetail(ex, message, server, null);
}
return(ex);
}
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
}
else
{
sb.Append("Timeout performing ").Append(message.CommandAndKey).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)");
}
void add(string lk, string sk, string v)
{
if (v != null)
{
data.Add(Tuple.Create(lk, v));
sb.Append(", ").Append(sk).Append(": ").Append(v);
}
}
if (server != null)
{
server.GetOutstandingCount(message.Command, out int inst, out int qs, out int @in);
add("Instantaneous", "inst", inst.ToString());
add("Queue-Awaiting-Response", "qs", qs.ToString());
if (@in >= 0)
{
add("Inbound-Bytes", "in", @in.ToString());
}
if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message.Command);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref mutiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log);
}
}
add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString());
}
add("Manager", "mgr", mutiplexer.SocketManager?.GetState());
add("Client-Name", "clientName", mutiplexer.ClientName);
var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString());
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
add("ThreadPool-IO-Completion", "IOCP", iocp);
add("ThreadPool-Workers", "WORKER", worker);
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (mutiplexer.IncludePerformanceCountersInExceptions)
{
add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
add("Version", "v", GetLibVersion());
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(")");
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
if (data != null)
{
foreach (var kv in data)
{
ex.Data["Redis-" + kv.Item1] = kv.Item2;
}
}
if (mutiplexer.IncludeDetailInExceptions)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
}
else
{
sb.Append("Timeout performing ").Append(message.CommandAndKey).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)");
}
void add(string lk, string sk, string v)
{
if (v != null)
{
data.Add(Tuple.Create(lk, v));
sb.Append(", ").Append(sk).Append(": ").Append(v);
}
}
// Add timeout data, if we have it
if (result == WriteResult.TimeoutBeforeWrite)
{
add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds));
try
{
if (message.TryGetPhysicalState(out var state, out var sentDelta, out var receivedDelta))
{
add("PhysicalState", "phys", state.ToString());
// these might not always be available
if (sentDelta >= 0)
{
add("OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB");
}
if (receivedDelta >= 0)
{
add("InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB");
}
}
}
catch { }
}
// Add server data, if we have it
if (server != null)
{
server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite);
add("OpsSinceLastHeartbeat", "inst", inst.ToString());
add("Queue-Awaiting-Write", "qu", qu.ToString());
add("Queue-Awaiting-Response", "qs", qs.ToString());
add("Active-Writer", "aw", aw.ToString());
if (@in >= 0)
{
add("Inbound-Bytes", "in", @in.ToString());
}
if (toRead >= 0)
{
add("Inbound-Pipe-Bytes", "in-pipe", toRead.ToString());
}
if (toWrite >= 0)
{
add("Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString());
}
if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message.Command);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref mutiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log);
}
}
add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString());
}
add("Manager", "mgr", mutiplexer.SocketManager?.GetState());
add("Client-Name", "clientName", mutiplexer.ClientName);
var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString());
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
add("ThreadPool-IO-Completion", "IOCP", iocp);
add("ThreadPool-Workers", "WORKER", worker);
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (mutiplexer.IncludePerformanceCountersInExceptions)
{
add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
add("Version", "v", GetLibVersion());
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(")");
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
if (data != null)
{
foreach (var kv in data)
{
ex.Data["Redis-" + kv.Item1] = kv.Item2;
}
}
if (mutiplexer.IncludeDetailInExceptions)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
if (message != null)
{
sb.Append(", command=").Append(message.Command); // no key here, note
}
}
else
{
sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)");
}
void add(string lk, string sk, string v)
{
if (v != null)
{
if (lk != null)
{
data.Add(Tuple.Create(lk, v));
}
if (sk != null)
{
sb.Append(", ").Append(sk).Append(": ").Append(v);
}
}
}
// Add timeout data, if we have it
if (result == WriteResult.TimeoutBeforeWrite)
{
add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds));
try
{
#if DEBUG
if (message.QueuePosition >= 0)
{
add("QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue
}
if ((int)message.ConnectionWriteState >= 0)
{
add("WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue
}
#endif
if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta))
{
add("Write-State", null, ws.ToString());
add("Read-State", null, rs.ToString());
// these might not always be available
if (sentDelta >= 0)
{
add("OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB");
}
if (receivedDelta >= 0)
{
add("InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB");
}
}
}
catch { }
}
if (message != null)
{
message.TryGetHeadMessages(out var now, out var next);
if (now != null)
{
add("Message-Current", "active", mutiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString());
}
if (next != null)
{
add("Message-Next", "next", mutiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString());
}
}
// Add server data, if we have it
if (server != null)
{
server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite, out var bs, out var rs, out var ws);
switch (rs)
{
case PhysicalConnection.ReadStatus.CompletePendingMessageAsync:
case PhysicalConnection.ReadStatus.CompletePendingMessageSync:
sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** ");
break;
}
add("OpsSinceLastHeartbeat", "inst", inst.ToString());
add("Queue-Awaiting-Write", "qu", qu.ToString());
add("Queue-Awaiting-Response", "qs", qs.ToString());
add("Active-Writer", "aw", aw.ToString());
if (qu != 0)
{
add("Backlog-Writer", "bw", bs.ToString());
}
if (rs != PhysicalConnection.ReadStatus.NA)
{
add("Read-State", "rs", rs.ToString());
}
if (ws != PhysicalConnection.WriteStatus.NA)
{
add("Write-State", "ws", ws.ToString());
}
if (@in >= 0)
{
add("Inbound-Bytes", "in", @in.ToString());
}
if (toRead >= 0)
{
add("Inbound-Pipe-Bytes", "in-pipe", toRead.ToString());
}
if (toWrite >= 0)
{
add("Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString());
}
if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message.Command);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref mutiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log);
}
}
add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString());
}
add("Manager", "mgr", mutiplexer.SocketManager?.GetState());
add("Client-Name", "clientName", mutiplexer.ClientName);
var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString());
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
add("ThreadPool-IO-Completion", "IOCP", iocp);
add("ThreadPool-Workers", "WORKER", worker);
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (mutiplexer.IncludePerformanceCountersInExceptions)
{
add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
add("Version", "v", GetLibVersion());
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(")");
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
if (data != null)
{
var exData = ex.Data;
foreach (var kv in data)
{
exData["Redis-" + kv.Item1] = kv.Item2;
}
}
if (mutiplexer.IncludeDetailInExceptions)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
