public ServerEndPoint Select(Message message)
{
if (message == null)
{
throw new ArgumentNullException("message");
}
int slot = NoSlot;
switch (serverType)
{
case ServerType.Cluster:
case ServerType.Twemproxy: // strictly speaking twemproxy uses a different hashing algo, but the hash-tag behavior is
// the same, so this does a pretty good job of spotting illegal commands before sending them
slot = message.GetHashSlot(this);
if (slot == MultipleSlots)
{
throw ExceptionFactory.MultiSlot(multiplexer.IncludeDetailInExceptions, message);
}
break;
}
return(Select(slot, message.Command, message.Flags));
}
private static void AddCommonDetail(
List <Tuple <string, string> > data,
StringBuilder sb,
Message message,
ConnectionMultiplexer multiplexer,
ServerEndPoint server
)
{
if (message != null)
{
message.TryGetHeadMessages(out var now, out var next);
if (now != null)
{
Add(data, sb, "Message-Current", "active", multiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString());
}
if (next != null)
{
Add(data, sb, "Message-Next", "next", multiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString());
}
}
// Add server data, if we have it
if (server != null && message != null)
{
var bs = server.GetBridgeStatus(message.IsForSubscriptionBridge ? ConnectionType.Subscription: ConnectionType.Interactive);
switch (bs.Connection.ReadStatus)
{
case PhysicalConnection.ReadStatus.CompletePendingMessageAsync:
case PhysicalConnection.ReadStatus.CompletePendingMessageSync:
sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** ");
break;
}
Add(data, sb, "OpsSinceLastHeartbeat", "inst", bs.MessagesSinceLastHeartbeat.ToString());
Add(data, sb, "Queue-Awaiting-Write", "qu", bs.BacklogMessagesPending.ToString());
Add(data, sb, "Queue-Awaiting-Response", "qs", bs.Connection.MessagesSentAwaitingResponse.ToString());
Add(data, sb, "Active-Writer", "aw", bs.IsWriterActive.ToString());
if (bs.BacklogMessagesPending != 0)
{
Add(data, sb, "Backlog-Writer", "bw", bs.BacklogStatus.ToString());
}
if (bs.Connection.ReadStatus != PhysicalConnection.ReadStatus.NA)
{
Add(data, sb, "Read-State", "rs", bs.Connection.ReadStatus.ToString());
}
if (bs.Connection.WriteStatus != PhysicalConnection.WriteStatus.NA)
{
Add(data, sb, "Write-State", "ws", bs.Connection.WriteStatus.ToString());
}
if (bs.Connection.BytesAvailableOnSocket >= 0)
{
Add(data, sb, "Inbound-Bytes", "in", bs.Connection.BytesAvailableOnSocket.ToString());
}
if (bs.Connection.BytesInReadPipe >= 0)
{
Add(data, sb, "Inbound-Pipe-Bytes", "in-pipe", bs.Connection.BytesInReadPipe.ToString());
}
if (bs.Connection.BytesInWritePipe >= 0)
{
Add(data, sb, "Outbound-Pipe-Bytes", "out-pipe", bs.Connection.BytesInWritePipe.ToString());
}
if (multiplexer.StormLogThreshold >= 0 && bs.Connection.MessagesSentAwaitingResponse >= multiplexer.StormLogThreshold && Interlocked.CompareExchange(ref multiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref multiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref multiplexer.stormLogSnapshot, log);
}
}
Add(data, sb, "Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/", ""));
}
Add(data, sb, "Multiplexer-Connects", "mc", $"{multiplexer._connectAttemptCount}/{multiplexer._connectCompletedCount}/{multiplexer._connectionCloseCount}");
Add(data, sb, "Manager", "mgr", multiplexer.SocketManager?.GetState());
Add(data, sb, "Client-Name", "clientName", multiplexer.ClientName);
if (message != null)
{
var hashSlot = message.GetHashSlot(multiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
Add(data, sb, "Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(multiplexer.ServerSelectionStrategy).ToString());
}
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker, out string workItems);
Add(data, sb, "ThreadPool-IO-Completion", "IOCP", iocp);
Add(data, sb, "ThreadPool-Workers", "WORKER", worker);
if (workItems != null)
{
Add(data, sb, "ThreadPool-Items", "POOL", workItems);
}
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (multiplexer.IncludePerformanceCountersInExceptions)
{
Add(data, sb, "Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
Add(data, sb, "Version", "v", GetLibVersion());
}
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
}
else
{
sb.Append("Timeout performing ").Append(message.CommandAndKey).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)");
}
void add(string lk, string sk, string v)
{
if (v != null)
{
data.Add(Tuple.Create(lk, v));
sb.Append(", ").Append(sk).Append(": ").Append(v);
}
}
// Add timeout data, if we have it
if (result == WriteResult.TimeoutBeforeWrite)
{
add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds));
try
{
if (message.TryGetPhysicalState(out var state, out var sentDelta, out var receivedDelta))
{
add("PhysicalState", "phys", state.ToString());
// these might not always be available
if (sentDelta >= 0)
{
add("OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB");
}
if (receivedDelta >= 0)
{
add("InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB");
}
}
}
catch { }
}
// Add server data, if we have it
if (server != null)
{
server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite);
add("OpsSinceLastHeartbeat", "inst", inst.ToString());
add("Queue-Awaiting-Write", "qu", qu.ToString());
add("Queue-Awaiting-Response", "qs", qs.ToString());
add("Active-Writer", "aw", aw.ToString());
if (@in >= 0)
{
add("Inbound-Bytes", "in", @in.ToString());
}
if (toRead >= 0)
{
add("Inbound-Pipe-Bytes", "in-pipe", toRead.ToString());
}
if (toWrite >= 0)
{
add("Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString());
}
if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message.Command);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref mutiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log);
}
}
add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString());
}
add("Manager", "mgr", mutiplexer.SocketManager?.GetState());
add("Client-Name", "clientName", mutiplexer.ClientName);
var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString());
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
add("ThreadPool-IO-Completion", "IOCP", iocp);
add("ThreadPool-Workers", "WORKER", worker);
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (mutiplexer.IncludePerformanceCountersInExceptions)
{
add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
add("Version", "v", GetLibVersion());
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(")");
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
if (data != null)
{
foreach (var kv in data)
{
ex.Data["Redis-" + kv.Item1] = kv.Item2;
}
}
if (mutiplexer.IncludeDetailInExceptions)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
public override int GetHashSlot(ServerSelectionStrategy serverSelectionStrategy)
{
return(tail.GetHashSlot(serverSelectionStrategy));
}
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
}
else
{
sb.Append("Timeout performing ").Append(message.CommandAndKey).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)");
}
void add(string lk, string sk, string v)
{
if (v != null)
{
data.Add(Tuple.Create(lk, v));
sb.Append(", ").Append(sk).Append(": ").Append(v);
}
}
if (server != null)
{
server.GetOutstandingCount(message.Command, out int inst, out int qs, out int @in);
add("Instantaneous", "inst", inst.ToString());
add("Queue-Awaiting-Response", "qs", qs.ToString());
if (@in >= 0)
{
add("Inbound-Bytes", "in", @in.ToString());
}
if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message.Command);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref mutiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log);
}
}
add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString());
}
add("Manager", "mgr", mutiplexer.SocketManager?.GetState());
add("Client-Name", "clientName", mutiplexer.ClientName);
var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString());
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
add("ThreadPool-IO-Completion", "IOCP", iocp);
add("ThreadPool-Workers", "WORKER", worker);
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (mutiplexer.IncludePerformanceCountersInExceptions)
{
add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
add("Version", "v", GetLibVersion());
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(")");
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
if (data != null)
{
foreach (var kv in data)
{
ex.Data["Redis-" + kv.Item1] = kv.Item2;
}
}
if (mutiplexer.IncludeDetailInExceptions)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
if (message != null)
{
sb.Append(", command=").Append(message.Command); // no key here, note
}
}
else
{
sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)");
}
void add(string lk, string sk, string v)
{
if (v != null)
{
if (lk != null)
{
data.Add(Tuple.Create(lk, v));
}
if (sk != null)
{
sb.Append(", ").Append(sk).Append(": ").Append(v);
}
}
}
// Add timeout data, if we have it
if (result == WriteResult.TimeoutBeforeWrite)
{
add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds));
try
{
#if DEBUG
if (message.QueuePosition >= 0)
{
add("QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue
}
if ((int)message.ConnectionWriteState >= 0)
{
add("WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue
}
#endif
if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta))
{
add("Write-State", null, ws.ToString());
add("Read-State", null, rs.ToString());
// these might not always be available
if (sentDelta >= 0)
{
add("OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB");
}
if (receivedDelta >= 0)
{
add("InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB");
}
}
}
catch { }
}
if (message != null)
{
message.TryGetHeadMessages(out var now, out var next);
if (now != null)
{
add("Message-Current", "active", mutiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString());
}
if (next != null)
{
add("Message-Next", "next", mutiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString());
}
}
// Add server data, if we have it
if (server != null)
{
server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite, out var bs, out var rs, out var ws);
switch (rs)
{
case PhysicalConnection.ReadStatus.CompletePendingMessageAsync:
case PhysicalConnection.ReadStatus.CompletePendingMessageSync:
sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** ");
break;
}
add("OpsSinceLastHeartbeat", "inst", inst.ToString());
add("Queue-Awaiting-Write", "qu", qu.ToString());
add("Queue-Awaiting-Response", "qs", qs.ToString());
add("Active-Writer", "aw", aw.ToString());
if (qu != 0)
{
add("Backlog-Writer", "bw", bs.ToString());
}
if (rs != PhysicalConnection.ReadStatus.NA)
{
add("Read-State", "rs", rs.ToString());
}
if (ws != PhysicalConnection.WriteStatus.NA)
{
add("Write-State", "ws", ws.ToString());
}
if (@in >= 0)
{
add("Inbound-Bytes", "in", @in.ToString());
}
if (toRead >= 0)
{
add("Inbound-Pipe-Bytes", "in-pipe", toRead.ToString());
}
if (toWrite >= 0)
{
add("Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString());
}
if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message.Command);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref mutiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log);
}
}
add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString());
}
add("Manager", "mgr", mutiplexer.SocketManager?.GetState());
add("Client-Name", "clientName", mutiplexer.ClientName);
var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString());
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
add("ThreadPool-IO-Completion", "IOCP", iocp);
add("ThreadPool-Workers", "WORKER", worker);
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (mutiplexer.IncludePerformanceCountersInExceptions)
{
add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
add("Version", "v", GetLibVersion());
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(")");
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
if (data != null)
{
var exData = ex.Data;
foreach (var kv in data)
{
exData["Redis-" + kv.Item1] = kv.Item2;
}
}
if (mutiplexer.IncludeDetailInExceptions)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
public override int GetHashSlot(ServerSelectionStrategy serverSelectionStrategy) => tail.GetHashSlot(serverSelectionStrategy);
