internal static Exception CommandDisabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
{
string s = GetLabel(includeDetail, command, message);
var ex = new RedisCommandException("This operation has been disabled in the command-map and cannot be used: " + s);
if (includeDetail) AddDetail(ex, message, server, s);
return ex;
}
private ProfileStorage(ConcurrentProfileStorageCollection pushTo, ServerEndPoint server, ProfileStorage resentFor, RetransmissionReasonType? reason)
{
PushToWhenFinished = pushTo;
OriginalProfiling = resentFor;
Server = server;
RetransmissionReason = reason;
}
internal static Exception MasterOnly(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
{
string s = GetLabel(includeDetail, command, message);
var ex = new RedisCommandException("Command cannot be issued to a slave: " + s);
if (includeDetail) AddDetail(ex, message, server, s);
return ex;
}
internal static Exception AdminModeNotEnabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
{
string s = GetLabel(includeDetail, command, message);
var ex = new RedisCommandException("This operation is not available unless admin mode is enabled: " + s);
if (includeDetail) AddDetail(ex, message, server, s);
return ex;
}
public PhysicalBridge(ServerEndPoint serverEndPoint, ConnectionType type)
{
ServerEndPoint = serverEndPoint;
ConnectionType = type;
Multiplexer = serverEndPoint.Multiplexer;
Name = Format.ToString(serverEndPoint.EndPoint) + "/" + ConnectionType.ToString();
completionManager = new CompletionManager(Multiplexer, Name);
}
public PhysicalBridge(ServerEndPoint serverEndPoint, ConnectionType type)
{
this.serverEndPoint = serverEndPoint;
this.connectionType = type;
this.multiplexer = serverEndPoint.Multiplexer;
this.Name = Format.ToString(serverEndPoint.EndPoint) + "/" + connectionType.ToString();
this.completionManager = new CompletionManager(multiplexer, Name);
}
internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server, ServerEndPoint[] serverSnapshot)
{
string s = GetLabel(includeDetail, command, message);
if (server != null)
{
//if we already have the serverEndpoint for connection failure use that
//otherwise it would output state of all the endpoints
serverSnapshot = new ServerEndPoint[] { server };
}
string exceptionmessage = "No connection is available to service this operation: " + s ;
var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, exceptionmessage, GetServerSnapshotInnerExceptions(serverSnapshot));
if (includeDetail)
{
AddDetail(ex, message, server, s);
}
return ex;
}
internal static Exception GetServerSnapshotInnerExceptions(ServerEndPoint[] serverSnapshot)
{
List<Exception> innerExceptions = new List<Exception>();
if (serverSnapshot != null)
{
for (int i = 0; i < serverSnapshot.Length; i++)
{
if (serverSnapshot[i].LastException != null)
{
var lastException = serverSnapshot[i].LastException;
innerExceptions.Add(lastException);
}
}
}
if (innerExceptions.Count == 1)
{
return innerExceptions[0];
}
else if(innerExceptions.Count > 1)
{
return new AggregateException(innerExceptions);
}
return null;
}
public bool TryResend(int hashSlot, Message message, EndPoint endpoint, bool isMoved)
{
try
{
if (serverType == ServerType.Standalone || hashSlot < 0 || hashSlot >= RedisClusterSlotCount)
{
return(false);
}
ServerEndPoint server = multiplexer.GetServerEndPoint(endpoint);
if (server != null)
{
bool retry = false;
if ((message.Flags & CommandFlags.NoRedirect) == 0)
{
message.SetAsking(!isMoved);
message.SetNoRedirect(); // once is enough
// note that everything so far is talking about MASTER nodes; we might be
// wanting a SLAVE, so we'll check
ServerEndPoint resendVia = null;
var command = message.Command;
switch (Message.GetMasterSlaveFlags(message.Flags))
{
case CommandFlags.DemandMaster:
resendVia = server.IsSelectable(command) ? server : null;
break;
case CommandFlags.PreferMaster:
resendVia = server.IsSelectable(command) ? server : FindSlave(server, command);
break;
case CommandFlags.PreferSlave:
resendVia = FindSlave(server, command) ?? (server.IsSelectable(command) ? server : null);
break;
case CommandFlags.DemandSlave:
resendVia = FindSlave(server, command);
break;
}
if (resendVia == null)
{
multiplexer.Trace("Unable to resend to " + endpoint);
}
else
{
message.PrepareToResend(resendVia, isMoved);
retry = resendVia.TryEnqueue(message);
}
}
if (isMoved) // update map; note we can still update the map even if we aren't actually goint to resend
{
var arr = MapForMutation();
var oldServer = arr[hashSlot];
arr[hashSlot] = server;
if (oldServer != server)
{
multiplexer.OnHashSlotMoved(hashSlot, oldServer == null ? null : oldServer.EndPoint, endpoint);
}
}
return(retry);
}
return(false);
}
catch
{
return(false);
}
}
public static ProfileStorage NewAttachedToSameContext(ProfileStorage resentFor, ServerEndPoint server, bool isMoved)
{
return new ProfileStorage(resentFor.PushToWhenFinished, server, resentFor, isMoved ? RetransmissionReasonType.Moved : RetransmissionReasonType.Ask);
}
internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, string message, ServerEndPoint server)
{
var ex = new RedisConnectionException(failureType, message);
if (includeDetail) AddDetail(ex, null, server, null);
return ex;
}
internal override T ExecuteSync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null) =>
throw new NotSupportedException("ExecuteSync cannot be used inside a batch");
internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
{
string s = GetLabel(includeDetail, command, message);
var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, "No connection is available to service this operation: " + s);
if (includeDetail)
{
AddDetail(ex, message, server, s);
}
return(ex);
}
internal static Exception CommandDisabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
{
string s = GetLabel(includeDetail, command, message);
var ex = new RedisCommandException("This operation has been disabled in the command-map and cannot be used: " + s);
if (includeDetail)
{
AddDetail(ex, message, server, s);
}
return(ex);
}
internal virtual T ExecuteSync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
{
if (message == null)
{
return(default(T)); // no-op
}
multiplexer.CheckMessage(message);
return(multiplexer.ExecuteSyncImpl <T>(message, processor, server));
}
internal void OnHeartbeat(bool ifConnectedOnly)
{
bool runThisTime = false;
try
{
runThisTime = !isDisposed && Interlocked.CompareExchange(ref beating, 1, 0) == 0;
if (!runThisTime)
{
return;
}
uint index = (uint)Interlocked.Increment(ref profileLogIndex);
long newSampleCount = Interlocked.Read(ref operationCount);
Interlocked.Exchange(ref profileLog[index % ProfileLogSamples], newSampleCount);
Interlocked.Exchange(ref profileLastLog, newSampleCount);
Trace("OnHeartbeat: " + (State)state);
switch (state)
{
case (int)State.Connecting:
int connectTimeMilliseconds = unchecked (Environment.TickCount - Thread.VolatileRead(ref connectStartTicks));
bool shouldRetry = Multiplexer.RawConfig.ReconnectRetryPolicy.ShouldRetry(Interlocked.Read(ref connectTimeoutRetryCount), connectTimeMilliseconds);
if (shouldRetry)
{
Interlocked.Increment(ref connectTimeoutRetryCount);
LastException = ExceptionFactory.UnableToConnect(Multiplexer, "ConnectTimeout");
Trace("Aborting connect");
// abort and reconnect
var snapshot = physical;
OnDisconnected(ConnectionFailureType.UnableToConnect, snapshot, out bool isCurrent, out State oldState);
using (snapshot) { } // dispose etc
TryConnect(null);
}
break;
case (int)State.ConnectedEstablishing:
case (int)State.ConnectedEstablished:
var tmp = physical;
if (tmp != null)
{
if (state == (int)State.ConnectedEstablished)
{
Interlocked.Exchange(ref connectTimeoutRetryCount, 0);
tmp.BridgeCouldBeNull?.ServerEndPoint?.ClearUnselectable(UnselectableFlags.DidNotRespond);
}
tmp.OnBridgeHeartbeat();
int writeEverySeconds = ServerEndPoint.WriteEverySeconds,
checkConfigSeconds = Multiplexer.RawConfig.ConfigCheckSeconds;
if (state == (int)State.ConnectedEstablished && ConnectionType == ConnectionType.Interactive &&
checkConfigSeconds > 0 && ServerEndPoint.LastInfoReplicationCheckSecondsAgo >= checkConfigSeconds &&
ServerEndPoint.CheckInfoReplication())
{
// that serves as a keep-alive, if it is accepted
}
else if (writeEverySeconds > 0 && tmp.LastWriteSecondsAgo >= writeEverySeconds)
{
Trace("OnHeartbeat - overdue");
if (state == (int)State.ConnectedEstablished)
{
KeepAlive();
}
else
{
OnDisconnected(ConnectionFailureType.SocketFailure, tmp, out bool ignore, out State oldState);
}
}
else if (writeEverySeconds <= 0 && tmp.IsIdle() &&
tmp.LastWriteSecondsAgo > 2 &&
tmp.GetSentAwaitingResponseCount() != 0)
{
// there's a chance this is a dead socket; sending data will shake that
// up a bit, so if we have an empty unsent queue and a non-empty sent
// queue, test the socket
KeepAlive();
}
}
break;
case (int)State.Disconnected:
Interlocked.Exchange(ref connectTimeoutRetryCount, 0);
if (!ifConnectedOnly)
{
Multiplexer.Trace("Resurrecting " + ToString());
Multiplexer.OnResurrecting(ServerEndPoint?.EndPoint, ConnectionType);
GetConnection(null);
}
break;
default:
Interlocked.Exchange(ref connectTimeoutRetryCount, 0);
break;
}
}
catch (Exception ex)
{
OnInternalError(ex);
Trace("OnHeartbeat error: " + ex.Message);
}
finally
{
if (runThisTime)
{
Interlocked.Exchange(ref beating, 0);
}
}
}
private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection connection, Message message)
{
if (message == null)
{
return(WriteResult.Success); // for some definition of success
}
bool isQueued = false;
try
{
var cmd = message.Command;
LastCommand = cmd;
bool isMasterOnly = message.IsMasterOnly();
if (isMasterOnly && ServerEndPoint.IsSlave && (ServerEndPoint.SlaveReadOnly || !ServerEndPoint.AllowSlaveWrites))
{
throw ExceptionFactory.MasterOnly(Multiplexer.IncludeDetailInExceptions, message.Command, message, ServerEndPoint);
}
switch (cmd)
{
case RedisCommand.QUIT:
connection.RecordQuit();
break;
case RedisCommand.EXEC:
Multiplexer.OnPreTransactionExec(message); // testing purposes, to force certain errors
break;
}
SelectDatabaseInsideWriteLock(connection, message);
if (!connection.TransactionActive)
{
var readmode = connection.GetReadModeCommand(isMasterOnly);
if (readmode != null)
{
connection.EnqueueInsideWriteLock(readmode);
readmode.WriteTo(connection);
readmode.SetRequestSent();
IncrementOpCount();
}
if (message.IsAsking)
{
var asking = ReusableAskingCommand;
connection.EnqueueInsideWriteLock(asking);
asking.WriteTo(connection);
asking.SetRequestSent();
IncrementOpCount();
}
}
switch (cmd)
{
case RedisCommand.WATCH:
case RedisCommand.MULTI:
connection.TransactionActive = true;
break;
case RedisCommand.UNWATCH:
case RedisCommand.EXEC:
case RedisCommand.DISCARD:
connection.TransactionActive = false;
break;
}
connection.EnqueueInsideWriteLock(message);
isQueued = true;
message.WriteTo(connection);
message.SetRequestSent();
IncrementOpCount();
// some commands smash our ability to trust the database; some commands
// demand an immediate flush
switch (cmd)
{
case RedisCommand.EVAL:
case RedisCommand.EVALSHA:
if (!ServerEndPoint.GetFeatures().ScriptingDatabaseSafe)
{
connection.SetUnknownDatabase();
}
break;
case RedisCommand.UNKNOWN:
case RedisCommand.DISCARD:
case RedisCommand.EXEC:
connection.SetUnknownDatabase();
break;
}
return(WriteResult.Success);
}
catch (RedisCommandException ex) when(!isQueued)
{
Trace("Write failed: " + ex.Message);
message.Fail(ConnectionFailureType.InternalFailure, ex, null);
this.CompleteSyncOrAsync(message);
// this failed without actually writing; we're OK with that... unless there's a transaction
if (connection?.TransactionActive == true)
{
// we left it in a broken state; need to kill the connection
connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, ex);
return(WriteResult.WriteFailure);
}
return(WriteResult.Success);
}
catch (Exception ex)
{
Trace("Write failed: " + ex.Message);
message.Fail(ConnectionFailureType.InternalFailure, ex, null);
this.CompleteSyncOrAsync(message);
// we're not sure *what* happened here; probably an IOException; kill the connection
connection?.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
return(WriteResult.WriteFailure);
}
}
public static ProfileStorage NewWithContext(ConcurrentProfileStorageCollection pushTo, ServerEndPoint server)
{
return(new ProfileStorage(pushTo, server, null, null));
}
public static ProfileStorage NewAttachedToSameContext(ProfileStorage resentFor, ServerEndPoint server, bool isMoved)
{
return(new ProfileStorage(resentFor.PushToWhenFinished, server, resentFor, isMoved ? RetransmissionReasonType.Moved : RetransmissionReasonType.Ask));
}
internal override Task <T> ExecuteAsync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
{
if (message == null)
{
return(CompletedTask <T> .Default(asyncState));
}
multiplexer.CheckMessage(message);
multiplexer.Trace("Wrapping " + message.Command, "Transaction");
// prepare the inner command as a task
Task <T> task;
if (message.IsFireAndForget)
{
task = CompletedTask <T> .Default(null); // F+F explicitly does not get async-state
}
else
{
var tcs = TaskSource.CreateDenyExecSync <T>(asyncState);
var source = ResultBox <T> .Get(tcs);
message.SetSource(source, processor);
task = tcs.Task;
}
// prepare an outer-command that decorates that, but expects QUEUED
var queued = new QueuedMessage(message);
var wasQueued = ResultBox <bool> .Get(null);
queued.SetSource(wasQueued, QueuedProcessor.Default);
// store it, and return the task of the *outer* command
// (there is no task for the inner command)
(pending ?? (pending = new List <QueuedMessage>())).Add(queued);
switch (message.Command)
{
case RedisCommand.UNKNOWN:
case RedisCommand.EVAL:
case RedisCommand.EVALSHA:
// people can do very naughty things in an EVAL
// including change the DB; change it back to what we
// think it should be!
var sel = PhysicalConnection.GetSelectDatabaseCommand(message.Db);
queued = new QueuedMessage(sel);
wasQueued = ResultBox <bool> .Get(null);
queued.SetSource(wasQueued, QueuedProcessor.Default);
pending.Add(queued);
break;
}
return(task);
}
internal virtual Task <T[]> ExecuteAsync <T>(Message[] messages, ResultProcessor <T> processor, ServerEndPoint server = null)
{
if (messages == null)
{
return(CompletedTask <T[]> .Default(asyncState));
}
foreach (var message in messages)
{
multiplexer.CheckMessage(message);
}
return(multiplexer.ExecuteAsyncImpl <T>(messages, processor, asyncState, server));
}
internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
}
else
{
sb.Append("Timeout performing ").Append(message.CommandAndKey).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)");
}
void add(string lk, string sk, string v)
{
if (v != null)
{
data.Add(Tuple.Create(lk, v));
sb.Append(", ").Append(sk).Append(": ").Append(v);
}
}
if (server != null)
{
server.GetOutstandingCount(message.Command, out int inst, out int qs, out int @in);
add("Instantaneous", "inst", inst.ToString());
add("Queue-Awaiting-Response", "qs", qs.ToString());
if (@in >= 0)
{
add("Inbound-Bytes", "in", @in.ToString());
}
if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message.Command);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref mutiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log);
}
}
add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString());
}
add("Manager", "mgr", mutiplexer.SocketManager?.GetState());
add("Client-Name", "clientName", mutiplexer.ClientName);
var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString());
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
add("ThreadPool-IO-Completion", "IOCP", iocp);
add("ThreadPool-Workers", "WORKER", worker);
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (mutiplexer.IncludePerformanceCountersInExceptions)
{
add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
add("Version", "v", GetLibVersion());
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(")");
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
if (data != null)
{
foreach (var kv in data)
{
ex.Data["Redis-" + kv.Item1] = kv.Item2;
}
}
if (mutiplexer.IncludeDetailInExceptions)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
internal static Exception NoConnectionAvailable(
ConnectionMultiplexer multiplexer,
Message message,
ServerEndPoint server,
ReadOnlySpan <ServerEndPoint> serverSnapshot = default,
RedisCommand command = default)
{
string commandLabel = GetLabel(multiplexer.IncludeDetailInExceptions, message?.Command ?? command, message);
if (server != null)
{
//if we already have the serverEndpoint for connection failure use that
//otherwise it would output state of all the endpoints
serverSnapshot = new ServerEndPoint[] { server };
}
var innerException = PopulateInnerExceptions(serverSnapshot == default ? multiplexer.GetServerSnapshot() : serverSnapshot);
// Try to get a useful error message for the user.
long attempts = multiplexer._connectAttemptCount, completions = multiplexer._connectCompletedCount;
string initialMessage;
// We only need to customize the connection if we're aborting on connect fail
// The "never" case would have thrown, if this was true
if (!multiplexer.RawConfig.AbortOnConnectFail && attempts <= multiplexer.RawConfig.ConnectRetry && completions == 0)
{
// Initial attempt, attempted use before an async connection completes
initialMessage = $"Connection to Redis never succeeded (attempts: {attempts} - connection likely in-progress), unable to service operation: ";
}
else if (!multiplexer.RawConfig.AbortOnConnectFail && attempts > multiplexer.RawConfig.ConnectRetry && completions == 0)
{
// Attempted use after a full initial retry connect count # of failures
// This can happen in Azure often, where user disables abort and has the wrong config
initialMessage = $"Connection to Redis never succeeded (attempts: {attempts} - check your config), unable to service operation: ";
}
else
{
// Default if we don't have a more useful error message here based on circumstances
initialMessage = "No connection is active/available to service this operation: ";
}
StringBuilder sb = new StringBuilder(initialMessage);
sb.Append(commandLabel);
string innermostExceptionstring = GetInnerMostExceptionMessage(innerException);
if (!string.IsNullOrEmpty(innermostExceptionstring))
{
sb.Append("; ").Append(innermostExceptionstring);
}
// Add counters and exception data if we have it
List <Tuple <string, string> > data = null;
if (multiplexer.IncludeDetailInExceptions)
{
data = new List <Tuple <string, string> >();
AddCommonDetail(data, sb, message, multiplexer, server);
}
var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown);
if (multiplexer.IncludeDetailInExceptions)
{
CopyDataToException(data, ex);
sb.Append("; ").Append(PerfCounterHelper.GetThreadPoolAndCPUSummary(multiplexer.IncludePerformanceCountersInExceptions));
AddExceptionDetail(ex, message, server, commandLabel);
}
return(ex);
}
internal RedisServer(ConnectionMultiplexer multiplexer, ServerEndPoint server, object asyncState) : base(multiplexer, asyncState)
{
this.server = server ?? throw new ArgumentNullException(nameof(server));
}
internal static Exception Timeout(bool includeDetail, string errorMessage, Message message, ServerEndPoint server)
{
var ex = new TimeoutException(errorMessage);
if (includeDetail)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
internal static Exception Timeout(bool includeDetail, string errorMessage, Message message, ServerEndPoint server)
{
var ex = new RedisTimeoutException(errorMessage, message?.Status ?? CommandStatus.Unknown);
if (includeDetail)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, string message, ServerEndPoint server)
{
var ex = new RedisConnectionException(failureType, message);
if (includeDetail)
{
AddDetail(ex, null, server, null);
}
return(ex);
}
internal static Exception TooManyArgs(bool includeDetail, string command, Message message, ServerEndPoint server, int required)
{
string s = GetLabel(includeDetail, command, message);
var ex = new RedisCommandException($"This operation would involve too many arguments ({required} vs the redis limit of {PhysicalConnection.REDIS_MAX_ARGS}): {s}");
if (includeDetail)
{
AddDetail(ex, message, server, s);
}
return(ex);
}
internal static Exception MasterOnly(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
{
string s = GetLabel(includeDetail, command, message);
var ex = new RedisCommandException("Command cannot be issued to a slave: " + s);
if (includeDetail)
{
AddDetail(ex, message, server, s);
}
return(ex);
}
internal override Task <T> ExecuteAsync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
{ // inject our expected server automatically
if (server == null)
{
server = this.server;
}
FixFlags(message, server);
if (!server.IsConnected)
{
if (message == null)
{
return(CompletedTask <T> .Default(asyncState));
}
if (message.IsFireAndForget)
{
return(CompletedTask <T> .Default(null)); // F+F explicitly does not get async-state
}
// no need to deny exec-sync here; will be complete before they see if
var tcs = TaskSource.Create <T>(asyncState);
ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server));
return(tcs.Task);
}
return(base.ExecuteAsync <T>(message, processor, server));
}
internal override Task <T> ExecuteAsync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
{
if (message == null)
{
return(CompletedTask <T> .Default(asyncState));
}
multiplexer.CheckMessage(message);
// prepare the inner command as a task
Task <T> task;
if (message.IsFireAndForget)
{
task = CompletedTask <T> .Default(null); // F+F explicitly does not get async-state
}
else
{
var source = TaskResultBox <T> .Create(out var tcs, asyncState);
task = tcs.Task;
message.SetSource(source, processor);
}
// store it
(pending ??= new List <Message>()).Add(message);
return(task);
}
internal override T ExecuteSync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
{ // inject our expected server automatically
if (server == null)
{
server = this.server;
}
FixFlags(message, server);
if (!server.IsConnected)
{
if (message == null || message.IsFireAndForget)
{
return(default(T));
}
throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
}
return(base.ExecuteSync <T>(message, processor, server));
}
public static ProfileStorage NewWithContext(ConcurrentProfileStorageCollection pushTo, ServerEndPoint server)
{
return new ProfileStorage(pushTo, server, null, null);
}
internal override RedisFeatures GetFeatures(int db, RedisKey key, CommandFlags flags, out ServerEndPoint server)
{
server = this.server;
return(new RedisFeatures(server.Version));
}
internal virtual RedisFeatures GetFeatures(int db, RedisKey key, CommandFlags flags, out ServerEndPoint server)
{
server = multiplexer.SelectServer(db, RedisCommand.PING, flags, key);
var version = server == null ? multiplexer.RawConfig.DefaultVersion : server.Version;
return(new RedisFeatures(version));
}
internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server, ServerEndPoint[] serverSnapshot)
{
string commandLabel = GetLabel(includeDetail, command, message);
if (server != null)
{
//if we already have the serverEndpoint for connection failure use that
//otherwise it would output state of all the endpoints
serverSnapshot = new ServerEndPoint[] { server };
}
var innerException = PopulateInnerExceptions(serverSnapshot);
StringBuilder exceptionmessage = new StringBuilder("No connection is available to service this operation: ").Append(commandLabel);
string innermostExceptionstring = GetInnerMostExceptionMessage(innerException);
if (!string.IsNullOrEmpty(innermostExceptionstring))
{
exceptionmessage.Append("; ").Append(innermostExceptionstring);
}
#if !CORE_CLR
if (includeDetail)
{
exceptionmessage.Append("; ").Append(ConnectionMultiplexer.GetThreadPoolAndCPUSummary());
}
#endif
var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, exceptionmessage.ToString(), innerException);
if (includeDetail)
{
AddDetail(ex, message, server, commandLabel);
}
return(ex);
}
internal static Exception DatabaseOutfRange(bool includeDetail, int targetDatabase, Message message, ServerEndPoint server)
{
var ex = new RedisCommandException("The database does not exist on the server: " + targetDatabase);
if (includeDetail) AddDetail(ex, message, server, null);
return ex;
}
internal static Exception DatabaseOutfRange(bool includeDetail, int targetDatabase, Message message, ServerEndPoint server)
{
var ex = new RedisCommandException("The database does not exist on the server: " + targetDatabase);
if (includeDetail)
{
AddDetail(ex, message, server, null);
}
return(ex);
}
internal static Exception AdminModeNotEnabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
{
string s = GetLabel(includeDetail, command, message);
var ex = new RedisCommandException("This operation is not available unless admin mode is enabled: " + s);
if (includeDetail)
{
AddDetail(ex, message, server, s);
}
return(ex);
}
private ServerEndPoint FindSlave(ServerEndPoint endpoint, RedisCommand command)
{
if (endpoint.IsSlave && endpoint.IsSelectable(command)) return endpoint;
var slaves = endpoint.Slaves;
for (int i = 0; i < slaves.Length; i++)
{
endpoint = slaves[i];
if (endpoint.IsSlave && endpoint.IsSelectable(command)) return endpoint;
}
return null;
}
internal virtual Task <T> ExecuteAsync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
{
if (message == null)
{
return(CompletedTask <T> .Default(asyncState));
}
multiplexer.CheckMessage(message);
return(multiplexer.ExecuteAsyncImpl <T>(message, processor, asyncState, server));
}
internal void UpdateClusterRange(int fromInclusive, int toInclusive, ServerEndPoint server)
{
var arr = MapForMutation();
for (int i = fromInclusive; i <= toInclusive; i++)
{
arr[i] = server;
}
}
private ServerEndPoint FindMaster(ServerEndPoint endpoint, RedisCommand command)
{
int max = 5;
do
{
if (!endpoint.IsSlave && endpoint.IsSelectable(command)) return endpoint;
endpoint = endpoint.Master;
} while (endpoint != null && --max != 0);
return null;
}
internal static Exception Timeout(ConnectionMultiplexer multiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null)
{
List <Tuple <string, string> > data = new List <Tuple <string, string> > {
Tuple.Create("Message", message.CommandAndKey)
};
var sb = new StringBuilder();
if (!string.IsNullOrEmpty(baseErrorMessage))
{
sb.Append(baseErrorMessage);
if (message != null)
{
sb.Append(", command=").Append(message.Command); // no key here, note
}
}
else
{
sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(multiplexer.TimeoutMilliseconds)).Append("ms)");
}
// Add timeout data, if we have it
if (result == WriteResult.TimeoutBeforeWrite)
{
Add(data, sb, "Timeout", "timeout", Format.ToString(multiplexer.TimeoutMilliseconds));
try
{
#if DEBUG
if (message.QueuePosition >= 0)
{
Add(data, sb, "QueuePosition", null, message.QueuePosition.ToString()); // the position the item was when added to the queue
}
if ((int)message.ConnectionWriteState >= 0)
{
Add(data, sb, "WriteState", null, message.ConnectionWriteState.ToString()); // what the physical was doing when it was added to the queue
}
#endif
if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta))
{
Add(data, sb, "Write-State", null, ws.ToString());
Add(data, sb, "Read-State", null, rs.ToString());
// these might not always be available
if (sentDelta >= 0)
{
Add(data, sb, "OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB");
}
if (receivedDelta >= 0)
{
Add(data, sb, "InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB");
}
}
}
catch { }
}
AddCommonDetail(data, sb, message, multiplexer, server);
sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
sb.Append(timeoutHelpLink);
sb.Append(')');
var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
{
HelpLink = timeoutHelpLink
};
CopyDataToException(data, ex);
if (multiplexer.IncludeDetailInExceptions)
{
AddExceptionDetail(ex, message, server, null);
}
return(ex);
}
private static void AddDetail(Exception exception, Message message, ServerEndPoint server, string label)
{
if (exception != null)
{
if (message != null) exception.Data.Add(DataCommandKey, message.CommandAndKey);
else if (label != null) exception.Data.Add(DataCommandKey, label);
if (server != null) exception.Data.Add(DataServerKey, Format.ToString(server.EndPoint));
}
}
private static void AddCommonDetail(
List <Tuple <string, string> > data,
StringBuilder sb,
Message message,
ConnectionMultiplexer multiplexer,
ServerEndPoint server
)
{
if (message != null)
{
message.TryGetHeadMessages(out var now, out var next);
if (now != null)
{
Add(data, sb, "Message-Current", "active", multiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString());
}
if (next != null)
{
Add(data, sb, "Message-Next", "next", multiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString());
}
}
// Add server data, if we have it
if (server != null && message != null)
{
server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite, out var bs, out var rs, out var ws);
switch (rs)
{
case PhysicalConnection.ReadStatus.CompletePendingMessageAsync:
case PhysicalConnection.ReadStatus.CompletePendingMessageSync:
sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** ");
break;
}
Add(data, sb, "OpsSinceLastHeartbeat", "inst", inst.ToString());
Add(data, sb, "Queue-Awaiting-Write", "qu", qu.ToString());
Add(data, sb, "Queue-Awaiting-Response", "qs", qs.ToString());
Add(data, sb, "Active-Writer", "aw", aw.ToString());
if (qu != 0)
{
Add(data, sb, "Backlog-Writer", "bw", bs.ToString());
}
if (rs != PhysicalConnection.ReadStatus.NA)
{
Add(data, sb, "Read-State", "rs", rs.ToString());
}
if (ws != PhysicalConnection.WriteStatus.NA)
{
Add(data, sb, "Write-State", "ws", ws.ToString());
}
if (@in >= 0)
{
Add(data, sb, "Inbound-Bytes", "in", @in.ToString());
}
if (toRead >= 0)
{
Add(data, sb, "Inbound-Pipe-Bytes", "in-pipe", toRead.ToString());
}
if (toWrite >= 0)
{
Add(data, sb, "Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString());
}
if (multiplexer.StormLogThreshold >= 0 && qs >= multiplexer.StormLogThreshold && Interlocked.CompareExchange(ref multiplexer.haveStormLog, 1, 0) == 0)
{
var log = server.GetStormLog(message.Command);
if (string.IsNullOrWhiteSpace(log))
{
Interlocked.Exchange(ref multiplexer.haveStormLog, 0);
}
else
{
Interlocked.Exchange(ref multiplexer.stormLogSnapshot, log);
}
}
Add(data, sb, "Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/", ""));
}
Add(data, sb, "Multiplexer-Connects", "mc", $"{multiplexer._connectAttemptCount}/{multiplexer._connectCompletedCount}/{multiplexer._connectionCloseCount}");
Add(data, sb, "Manager", "mgr", multiplexer.SocketManager?.GetState());
Add(data, sb, "Client-Name", "clientName", multiplexer.ClientName);
if (message != null)
{
var hashSlot = message.GetHashSlot(multiplexer.ServerSelectionStrategy);
// only add keyslot if its a valid cluster key slot
if (hashSlot != ServerSelectionStrategy.NoSlot)
{
Add(data, sb, "Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(multiplexer.ServerSelectionStrategy).ToString());
}
}
int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);
Add(data, sb, "ThreadPool-IO-Completion", "IOCP", iocp);
Add(data, sb, "ThreadPool-Workers", "WORKER", worker);
data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));
if (multiplexer.IncludePerformanceCountersInExceptions)
{
Add(data, sb, "Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
}
Add(data, sb, "Version", "v", GetLibVersion());
}
internal static Exception Timeout(bool includeDetail, string errorMessage, Message message, ServerEndPoint server)
{
var ex = new TimeoutException(errorMessage);
if (includeDetail) AddDetail(ex, message, server, null);
return ex;
}
internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
{
string s = GetLabel(includeDetail, command, message);
var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, "No connection is available to service this operation: " + s);
if (includeDetail) AddDetail(ex, message, server, s);
return ex;
}