public override string ToString() => Format.ToString(EndPoint);
public override string ToString() => ConnectionType + "/" + Format.ToString(ServerEndPoint.EndPoint);
internal static Exception Timeout(ConnectionMultiplexer multiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
if (message != null)
{
sb.Append(", command=").Append(message.Command); // no key here, note
}
}
else
{
sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(multiplexer.TimeoutMilliseconds)).Append("ms)");
}
// Add timeout data, if we have it
if (result == WriteResult.TimeoutBeforeWrite)
{
Add(data, sb, "Timeout", "timeout", Format.ToString(multiplexer.TimeoutMilliseconds));
try
{
#if DEBUG
if (message.QueuePosition >= 0)
{
Add(data, sb, "QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue
}
if ((int)message.ConnectionWriteState >= 0)
{
Add(data, sb, "WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue
}
#endif
if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta))
{
Add(data, sb, "Write-State", null, ws.ToString());
Add(data, sb, "Read-State", null, rs.ToString());
// these might not always be available
if (sentDelta >= 0)
{
Add(data, sb, "OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB");
}
if (receivedDelta >= 0)
{
Add(data, sb, "InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB");
}
}
}
catch { }
}
AddCommonDetail(data, sb, message, multiplexer, server);
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(")");
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
CopyDataToException(data, ex);
if (multiplexer.IncludeDetailInExceptions)
{
AddExceptionDetail(ex, message, server, null);
}
return(ex);
}
public PhysicalConnection(PhysicalBridge bridge)
{
lastWriteTickCount = lastReadTickCount = Environment.TickCount;
lastBeatTickCount = 0;
connectionType = bridge.ConnectionType;
Multiplexer = bridge.Multiplexer;
ChannelPrefix = Multiplexer.RawConfig.ChannelPrefix;
if (ChannelPrefix?.Length == 0)
{
ChannelPrefix = null; // null tests are easier than null+empty
}
var endpoint = bridge.ServerEndPoint.EndPoint;
physicalName = connectionType + "#" + Interlocked.Increment(ref totalCount) + "@" + Format.ToString(endpoint);
Bridge = bridge;
OnCreateEcho();
}
private void MatchResult(RawResult result)
{
// check to see if it could be an out-of-band pubsub message
if (connectionType == ConnectionType.Subscription && result.Type == ResultType.MultiBulk)
{ // out of band message does not match to a queued message
var items = result.GetItems();
if (items.Length >= 3 && items[0].IsEqual(message))
{
// special-case the configuration change broadcasts (we don't keep that in the usual pub/sub registry)
var configChanged = Multiplexer.ConfigurationChangedChannel;
if (configChanged != null && items[1].IsEqual(configChanged))
{
EndPoint blame = null;
try
{
if (!items[2].IsEqual(RedisLiterals.ByteWildcard))
{
blame = Format.TryParseEndPoint(items[2].GetString());
}
}
catch { /* no biggie */ }
Multiplexer.Trace("Configuration changed: " + Format.ToString(blame), physicalName);
Multiplexer.ReconfigureIfNeeded(blame, true, "broadcast");
}
// invoke the handlers
var channel = items[1].AsRedisChannel(ChannelPrefix, RedisChannel.PatternMode.Literal);
Multiplexer.Trace("MESSAGE: " + channel, physicalName);
if (!channel.IsNull)
{
Multiplexer.OnMessage(channel, channel, items[2].AsRedisValue());
}
return; // AND STOP PROCESSING!
}
else if (items.Length >= 4 && items[0].IsEqual(pmessage))
{
var channel = items[2].AsRedisChannel(ChannelPrefix, RedisChannel.PatternMode.Literal);
Multiplexer.Trace("PMESSAGE: " + channel, physicalName);
if (!channel.IsNull)
{
var sub = items[1].AsRedisChannel(ChannelPrefix, RedisChannel.PatternMode.Pattern);
Multiplexer.OnMessage(sub, channel, items[3].AsRedisValue());
}
return; // AND STOP PROCESSING!
}
// if it didn't look like "[p]message", then we still need to process the pending queue
}
Multiplexer.Trace("Matching result...", physicalName);
Message msg;
lock (outstanding)
{
Multiplexer.Trace(outstanding.Count == 0, "Nothing to respond to!", physicalName);
msg = outstanding.Dequeue();
}
Multiplexer.Trace("Response to: " + msg, physicalName);
if (msg.ComputeResult(this, result))
{
Bridge.CompleteSyncOrAsync(msg);
}
}
public void RecordConnectionFailed(ConnectionFailureType failureType, ref SocketManager.ManagerState managerState, Exception innerException = null, [CallerMemberName] string origin = null)
{
IdentifyFailureType(innerException, ref failureType);
managerState = SocketManager.ManagerState.RecordConnectionFailed_OnInternalError;
if (failureType == ConnectionFailureType.InternalFailure)
{
OnInternalError(innerException, origin);
}
// stop anything new coming in...
Bridge.Trace("Failed: " + failureType);
int @in = -1, ar = -1;
managerState = SocketManager.ManagerState.RecordConnectionFailed_OnDisconnected;
Bridge.OnDisconnected(failureType, this, out bool isCurrent, out PhysicalBridge.State oldState);
if (oldState == PhysicalBridge.State.ConnectedEstablished)
{
try
{
@in = GetAvailableInboundBytes(out ar);
}
catch { /* best effort only */ }
}
if (isCurrent && Interlocked.CompareExchange(ref failureReported, 1, 0) == 0)
{
managerState = SocketManager.ManagerState.RecordConnectionFailed_ReportFailure;
int now = Environment.TickCount, lastRead = Thread.VolatileRead(ref lastReadTickCount), lastWrite = Thread.VolatileRead(ref lastWriteTickCount),
lastBeat = Thread.VolatileRead(ref lastBeatTickCount);
int unansweredRead = Thread.VolatileRead(ref firstUnansweredWriteTickCount);
var exMessage = new StringBuilder(failureType.ToString());
var data = new List <Tuple <string, string> >();
if (Multiplexer.IncludeDetailInExceptions)
{
exMessage.Append(" on " + Format.ToString(Bridge.ServerEndPoint.EndPoint) + "/" + connectionType);
data.Add(Tuple.Create("FailureType", failureType.ToString()));
data.Add(Tuple.Create("EndPoint", Format.ToString(Bridge.ServerEndPoint.EndPoint)));
void add(string lk, string sk, string v)
{
data.Add(Tuple.Create(lk, v));
exMessage.Append(", ").Append(sk).Append(": ").Append(v);
}
add("Origin", "origin", origin);
add("Input-Buffer", "input-buffer", ioBufferBytes.ToString());
add("Outstanding-Responses", "outstanding", GetSentAwaitingResponseCount().ToString());
add("Last-Read", "last-read", (unchecked (now - lastRead) / 1000) + "s ago");
add("Last-Write", "last-write", (unchecked (now - lastWrite) / 1000) + "s ago");
add("Unanswered-Write", "unanswered-write", (unchecked (now - unansweredRead) / 1000) + "s ago");
add("Keep-Alive", "keep-alive", Bridge.ServerEndPoint.WriteEverySeconds + "s");
add("Pending", "pending", Bridge.GetPendingCount().ToString());
add("Previous-Physical-State", "state", oldState.ToString());
if (@in >= 0)
{
add("Inbound-Bytes", "in", @in.ToString());
add("Active-Readers", "ar", ar.ToString());
}
add("Last-Heartbeat", "last-heartbeat", (lastBeat == 0 ? "never" : ((unchecked (now - lastBeat) / 1000) + "s ago")) + (Bridge.IsBeating ? " (mid-beat)" : ""));
add("Last-Multiplexer-Heartbeat", "last-mbeat", Multiplexer.LastHeartbeatSecondsAgo + "s ago");
add("Last-Global-Heartbeat", "global", ConnectionMultiplexer.LastGlobalHeartbeatSecondsAgo + "s ago");
#if FEATURE_SOCKET_MODE_POLL
var mgr = Bridge.Multiplexer.SocketManager;
add("SocketManager-State", "mgr", mgr.State.ToString());
add("Last-Error", "err", mgr.LastErrorTimeRelative());
#endif
}
var ex = innerException == null
? new RedisConnectionException(failureType, exMessage.ToString())
: new RedisConnectionException(failureType, exMessage.ToString(), innerException);
foreach (var kv in data)
{
ex.Data["Redis-" + kv.Item1] = kv.Item2;
}
managerState = SocketManager.ManagerState.RecordConnectionFailed_OnConnectionFailed;
Bridge.OnConnectionFailed(this, failureType, ex);
}
// cleanup
managerState = SocketManager.ManagerState.RecordConnectionFailed_FailOutstanding;
lock (outstanding)
{
Bridge.Trace(outstanding.Count != 0, "Failing outstanding messages: " + outstanding.Count);
while (outstanding.Count != 0)
{
var next = outstanding.Dequeue();
Bridge.Trace("Failing: " + next);
next.Fail(failureType, innerException);
Bridge.CompleteSyncOrAsync(next);
}
}
// burn the socket
managerState = SocketManager.ManagerState.RecordConnectionFailed_ShutdownSocket;
Multiplexer.SocketManager?.Shutdown(socketToken);
}
private static void WriteRaw(Stream stream, long value, bool withLengthPrefix = false)
{
if (value >= 0 && value <= 9)
{
if (withLengthPrefix)
{
stream.WriteByte((byte)'1');
stream.Write(Crlf, 0, 2);
}
stream.WriteByte((byte)((int)'0' + (int)value));
}
else if (value >= 10 && value < 100)
{
if (withLengthPrefix)
{
stream.WriteByte((byte)'2');
stream.Write(Crlf, 0, 2);
}
stream.WriteByte((byte)((int)'0' + ((int)value / 10)));
stream.WriteByte((byte)((int)'0' + ((int)value % 10)));
}
else if (value >= 100 && value < 1000)
{
int v = (int)value;
int units = v % 10;
v /= 10;
int tens = v % 10, hundreds = v / 10;
if (withLengthPrefix)
{
stream.WriteByte((byte)'3');
stream.Write(Crlf, 0, 2);
}
stream.WriteByte((byte)((int)'0' + hundreds));
stream.WriteByte((byte)((int)'0' + tens));
stream.WriteByte((byte)((int)'0' + units));
}
else if (value < 0 && value >= -9)
{
if (withLengthPrefix)
{
stream.WriteByte((byte)'2');
stream.Write(Crlf, 0, 2);
}
stream.WriteByte((byte)'-');
stream.WriteByte((byte)((int)'0' - (int)value));
}
else if (value <= -10 && value > -100)
{
if (withLengthPrefix)
{
stream.WriteByte((byte)'3');
stream.Write(Crlf, 0, 2);
}
value = -value;
stream.WriteByte((byte)'-');
stream.WriteByte((byte)((int)'0' + ((int)value / 10)));
stream.WriteByte((byte)((int)'0' + ((int)value % 10)));
}
else
{
var bytes = Encoding.ASCII.GetBytes(Format.ToString(value));
if (withLengthPrefix)
{
WriteRaw(stream, bytes.Length, false);
}
stream.Write(bytes, 0, bytes.Length);
}
stream.Write(Crlf, 0, 2);
}
public void RecordConnectionFailed(ConnectionFailureType failureType, Exception innerException = null, [CallerMemberName] string origin = null)
{
IdentifyFailureType(innerException, ref failureType);
if (failureType == ConnectionFailureType.InternalFailure)
{
OnInternalError(innerException, origin);
}
// stop anything new coming in...
bridge.Trace("Failed: " + failureType);
bool isCurrent;
PhysicalBridge.State oldState;
bridge.OnDisconnected(failureType, this, out isCurrent, out oldState);
if (isCurrent && Interlocked.CompareExchange(ref failureReported, 1, 0) == 0)
{
//try
//{
int now = Environment.TickCount, lastRead = Thread.VolatileRead(ref lastReadTickCount), lastWrite = Thread.VolatileRead(ref lastWriteTickCount),
lastBeat = Thread.VolatileRead(ref lastBeatTickCount);
string message = failureType + " on " + Format.ToString(bridge.ServerEndPoint.EndPoint) + "/" + connectionType
+ ", input-buffer: " + ioBufferBytes + ", outstanding: " + GetSentAwaitingResponseCount()
+ ", last-read: " + unchecked (now - lastRead) / 1000 + "s ago, last-write: " + unchecked (now - lastWrite) / 1000 + "s ago, keep-alive: " + bridge.ServerEndPoint.WriteEverySeconds + "s, pending: "
+ bridge.GetPendingCount() + ", state: " + oldState + ", last-heartbeat: " + (lastBeat == 0 ? "never" : (unchecked (now - lastBeat) / 1000 + "s ago"))
+ (bridge.IsBeating ? " (mid-beat)" : "") + ", last-mbeat: " + multiplexer.LastHeartbeatSecondsAgo + "s ago, global: "
+ ConnectionMultiplexer.LastGlobalHeartbeatSecondsAgo + "s ago";
var ex = innerException == null
? new RedisConnectionException(failureType, message)
: new RedisConnectionException(failureType, message, innerException);
bridge.OnConnectionFailed(this, failureType, ex);
// throw ex;
//}
//catch (Exception caught)
//{
// bridge.OnConnectionFailed(this, failureType, caught);
//}
}
// cleanup
lock (outstanding)
{
bridge.Trace(outstanding.Count != 0, "Failing outstanding messages: " + outstanding.Count);
while (outstanding.Count != 0)
{
var next = outstanding.Dequeue();
bridge.Trace("Failing: " + next);
next.Fail(failureType, innerException);
bridge.CompleteSyncOrAsync(next);
}
}
// burn the socket
var socketManager = multiplexer.SocketManager;
if (socketManager != null)
{
socketManager.Shutdown(socketToken);
}
}
public override string ToString()
{
return(Format.ToString(EndPoint));
}
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
}
else
{
sb.Append("Timeout performing ").Append(message.CommandAndKey).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)");
}
void add(string lk, string sk, string v)
{
if (v != null)
{
data.Add(Tuple.Create(lk, v));
sb.Append(", ").Append(sk).Append(": ").Append(v);
}
}
if (server != null)
{
server.GetOutstandingCount(message.Command, out int inst, out int qs, out int @in);
add("Instantaneous", "inst", inst.ToString());
add("Queue-Awaiting-Response", "qs", qs.ToString());
add("Inbound-Bytes", "in", @in.ToString());
if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message.Command);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref mutiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log);
}
}
add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString());
}
add("Manager", "mgr", mutiplexer.SocketManager?.GetState());
add("Client-Name", "clientName", mutiplexer.ClientName);
var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString());
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
add("ThreadPool-IO-Completion", "IOCP", iocp);
add("ThreadPool-Workers", "WORKER", worker);
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (mutiplexer.IncludePerformanceCountersInExceptions)
{
add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(")");
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
if (data != null)
{
foreach (var kv in data)
{
ex.Data["Redis-" + kv.Item1] = kv.Item2;
}
}
if (mutiplexer.IncludeDetailInExceptions)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
if (message != null)
{
sb.Append(", command=").Append(message.Command); // no key here, note
}
}
else
{
sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)");
}
void add(string lk, string sk, string v)
{
if (v != null)
{
if (lk != null)
{
data.Add(Tuple.Create(lk, v));
}
if (sk != null)
{
sb.Append(", ").Append(sk).Append(": ").Append(v);
}
}
}
// Add timeout data, if we have it
if (result == WriteResult.TimeoutBeforeWrite)
{
add("Timeout", "timeout", Format.ToString(mutiplexer.TimeoutMilliseconds));
try
{
#if DEBUG
if (message.QueuePosition >= 0)
{
add("QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue
}
if ((int)message.ConnectionWriteState >= 0)
{
add("WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue
}
#endif
if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta))
{
add("Write-State", null, ws.ToString());
add("Read-State", null, rs.ToString());
// these might not always be available
if (sentDelta >= 0)
{
add("OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB");
}
if (receivedDelta >= 0)
{
add("InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB");
}
}
}
catch { }
}
if (message != null)
{
message.TryGetHeadMessages(out var now, out var next);
if (now != null)
{
add("Message-Current", "active", mutiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString());
}
if (next != null)
{
add("Message-Next", "next", mutiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString());
}
}
// Add server data, if we have it
if (server != null)
{
server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite, out var bs, out var rs, out var ws);
switch (rs)
{
case PhysicalConnection.ReadStatus.CompletePendingMessageAsync:
case PhysicalConnection.ReadStatus.CompletePendingMessageSync:
sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** ");
break;
}
add("OpsSinceLastHeartbeat", "inst", inst.ToString());
add("Queue-Awaiting-Write", "qu", qu.ToString());
add("Queue-Awaiting-Response", "qs", qs.ToString());
add("Active-Writer", "aw", aw.ToString());
if (qu != 0)
{
add("Backlog-Writer", "bw", bs.ToString());
}
if (rs != PhysicalConnection.ReadStatus.NA)
{
add("Read-State", "rs", rs.ToString());
}
if (ws != PhysicalConnection.WriteStatus.NA)
{
add("Write-State", "ws", ws.ToString());
}
if (@in >= 0)
{
add("Inbound-Bytes", "in", @in.ToString());
}
if (toRead >= 0)
{
add("Inbound-Pipe-Bytes", "in-pipe", toRead.ToString());
}
if (toWrite >= 0)
{
add("Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString());
}
if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message.Command);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref mutiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log);
}
}
add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/", ""));
}
add("Manager", "mgr", mutiplexer.SocketManager?.GetState());
add("Client-Name", "clientName", mutiplexer.ClientName);
var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString());
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
add("ThreadPool-IO-Completion", "IOCP", iocp);
add("ThreadPool-Workers", "WORKER", worker);
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (mutiplexer.IncludePerformanceCountersInExceptions)
{
add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
add("Version", "v", GetLibVersion());
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(")");
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
if (data != null)
{
var exData = ex.Data;
foreach (var kv in data)
{
exData["Redis-" + kv.Item1] = kv.Item2;
}
}
if (mutiplexer.IncludeDetailInExceptions)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
public override string ToString()
{
return(connectionType + "/" + Format.ToString(serverEndPoint.EndPoint));
}
