internal static Exception CommandDisabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
 {
     string s = GetLabel(includeDetail, command, message);
     var ex = new RedisCommandException("This operation has been disabled in the command-map and cannot be used: " + s);
     if (includeDetail) AddDetail(ex, message, server, s);
     return ex;
 }
 private ProfileStorage(ConcurrentProfileStorageCollection pushTo, ServerEndPoint server, ProfileStorage resentFor, RetransmissionReasonType? reason)
 {
     PushToWhenFinished = pushTo;
     OriginalProfiling = resentFor;
     Server = server;
     RetransmissionReason = reason;
 }
 internal static Exception MasterOnly(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
 {
     string s = GetLabel(includeDetail, command, message);
     var ex = new RedisCommandException("Command cannot be issued to a slave: " + s);
     if (includeDetail) AddDetail(ex, message, server, s);
     return ex;
 }
 internal static Exception AdminModeNotEnabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
 {
     string s = GetLabel(includeDetail, command, message);
     var ex = new RedisCommandException("This operation is not available unless admin mode is enabled: " + s);
     if (includeDetail) AddDetail(ex, message, server, s);
     return ex;
 }
 public PhysicalBridge(ServerEndPoint serverEndPoint, ConnectionType type)
 {
     ServerEndPoint = serverEndPoint;
     ConnectionType = type;
     Multiplexer = serverEndPoint.Multiplexer;
     Name = Format.ToString(serverEndPoint.EndPoint) + "/" + ConnectionType.ToString();
     completionManager = new CompletionManager(Multiplexer, Name);
 }
 public PhysicalBridge(ServerEndPoint serverEndPoint, ConnectionType type)
 {
     this.serverEndPoint = serverEndPoint;
     this.connectionType = type;
     this.multiplexer = serverEndPoint.Multiplexer;
     this.Name = Format.ToString(serverEndPoint.EndPoint) + "/" + connectionType.ToString();
     this.completionManager = new CompletionManager(multiplexer, Name);
 }
        internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server, ServerEndPoint[] serverSnapshot)
        {
            string s = GetLabel(includeDetail, command, message);

            if (server != null)
            {
                //if we already have the serverEndpoint for connection failure use that
                //otherwise it would output state of all the endpoints
                serverSnapshot = new ServerEndPoint[] { server };
            }
            string exceptionmessage = "No connection is available to service this operation: " + s ;
            var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, exceptionmessage, GetServerSnapshotInnerExceptions(serverSnapshot));
            if (includeDetail)
            {
                AddDetail(ex, message, server, s);
            }
            return ex;
        }
Esempio n. 8
0
 internal static Exception GetServerSnapshotInnerExceptions(ServerEndPoint[] serverSnapshot)
 {
     List<Exception> innerExceptions = new List<Exception>();
     if (serverSnapshot != null)
     {
         for (int i = 0; i < serverSnapshot.Length; i++)
         {
             if (serverSnapshot[i].LastException != null)
             {
                 var lastException = serverSnapshot[i].LastException;
                 innerExceptions.Add(lastException);
             }
         }
     }
     if (innerExceptions.Count == 1)
     {
         return innerExceptions[0];
     }
     else if(innerExceptions.Count > 1)
     {
         return new AggregateException(innerExceptions);
     }
     return null;
 }
        public bool TryResend(int hashSlot, Message message, EndPoint endpoint, bool isMoved)
        {
            try
            {
                if (serverType == ServerType.Standalone || hashSlot < 0 || hashSlot >= RedisClusterSlotCount)
                {
                    return(false);
                }

                ServerEndPoint server = multiplexer.GetServerEndPoint(endpoint);
                if (server != null)
                {
                    bool retry = false;
                    if ((message.Flags & CommandFlags.NoRedirect) == 0)
                    {
                        message.SetAsking(!isMoved);
                        message.SetNoRedirect(); // once is enough

                        // note that everything so far is talking about MASTER nodes; we might be
                        // wanting a SLAVE, so we'll check
                        ServerEndPoint resendVia = null;
                        var            command   = message.Command;
                        switch (Message.GetMasterSlaveFlags(message.Flags))
                        {
                        case CommandFlags.DemandMaster:
                            resendVia = server.IsSelectable(command) ? server : null;
                            break;

                        case CommandFlags.PreferMaster:
                            resendVia = server.IsSelectable(command) ? server : FindSlave(server, command);
                            break;

                        case CommandFlags.PreferSlave:
                            resendVia = FindSlave(server, command) ?? (server.IsSelectable(command) ? server : null);
                            break;

                        case CommandFlags.DemandSlave:
                            resendVia = FindSlave(server, command);
                            break;
                        }
                        if (resendVia == null)
                        {
                            multiplexer.Trace("Unable to resend to " + endpoint);
                        }
                        else
                        {
                            message.PrepareToResend(resendVia, isMoved);
                            retry = resendVia.TryEnqueue(message);
                        }
                    }

                    if (isMoved) // update map; note we can still update the map even if we aren't actually goint to resend
                    {
                        var arr       = MapForMutation();
                        var oldServer = arr[hashSlot];
                        arr[hashSlot] = server;
                        if (oldServer != server)
                        {
                            multiplexer.OnHashSlotMoved(hashSlot, oldServer == null ? null : oldServer.EndPoint, endpoint);
                        }
                    }

                    return(retry);
                }
                return(false);
            }
            catch
            {
                return(false);
            }
        }
 public static ProfileStorage NewAttachedToSameContext(ProfileStorage resentFor, ServerEndPoint server, bool isMoved)
 {
     return new ProfileStorage(resentFor.PushToWhenFinished, server, resentFor, isMoved ? RetransmissionReasonType.Moved : RetransmissionReasonType.Ask);
 }
 internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, string message, ServerEndPoint server)
 {
     var ex = new RedisConnectionException(failureType, message);
     if (includeDetail) AddDetail(ex, null, server, null);
     return ex;
 }
Esempio n. 12
0
 internal override T ExecuteSync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null) =>
 throw new NotSupportedException("ExecuteSync cannot be used inside a batch");
Esempio n. 13
0
        internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
        {
            string s  = GetLabel(includeDetail, command, message);
            var    ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, "No connection is available to service this operation: " + s);

            if (includeDetail)
            {
                AddDetail(ex, message, server, s);
            }
            return(ex);
        }
        internal static Exception CommandDisabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
        {
            string s  = GetLabel(includeDetail, command, message);
            var    ex = new RedisCommandException("This operation has been disabled in the command-map and cannot be used: " + s);

            if (includeDetail)
            {
                AddDetail(ex, message, server, s);
            }
            return(ex);
        }
Esempio n. 15
0
 internal virtual T ExecuteSync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
 {
     if (message == null)
     {
         return(default(T));                 // no-op
     }
     multiplexer.CheckMessage(message);
     return(multiplexer.ExecuteSyncImpl <T>(message, processor, server));
 }
        internal void OnHeartbeat(bool ifConnectedOnly)
        {
            bool runThisTime = false;

            try
            {
                runThisTime = !isDisposed && Interlocked.CompareExchange(ref beating, 1, 0) == 0;
                if (!runThisTime)
                {
                    return;
                }

                uint index          = (uint)Interlocked.Increment(ref profileLogIndex);
                long newSampleCount = Interlocked.Read(ref operationCount);
                Interlocked.Exchange(ref profileLog[index % ProfileLogSamples], newSampleCount);
                Interlocked.Exchange(ref profileLastLog, newSampleCount);
                Trace("OnHeartbeat: " + (State)state);
                switch (state)
                {
                case (int)State.Connecting:
                    int  connectTimeMilliseconds = unchecked (Environment.TickCount - Thread.VolatileRead(ref connectStartTicks));
                    bool shouldRetry             = Multiplexer.RawConfig.ReconnectRetryPolicy.ShouldRetry(Interlocked.Read(ref connectTimeoutRetryCount), connectTimeMilliseconds);
                    if (shouldRetry)
                    {
                        Interlocked.Increment(ref connectTimeoutRetryCount);
                        LastException = ExceptionFactory.UnableToConnect(Multiplexer, "ConnectTimeout");
                        Trace("Aborting connect");
                        // abort and reconnect
                        var snapshot = physical;
                        OnDisconnected(ConnectionFailureType.UnableToConnect, snapshot, out bool isCurrent, out State oldState);
                        using (snapshot) { }     // dispose etc
                        TryConnect(null);
                    }
                    break;

                case (int)State.ConnectedEstablishing:
                case (int)State.ConnectedEstablished:
                    var tmp = physical;
                    if (tmp != null)
                    {
                        if (state == (int)State.ConnectedEstablished)
                        {
                            Interlocked.Exchange(ref connectTimeoutRetryCount, 0);
                            tmp.BridgeCouldBeNull?.ServerEndPoint?.ClearUnselectable(UnselectableFlags.DidNotRespond);
                        }
                        tmp.OnBridgeHeartbeat();
                        int writeEverySeconds  = ServerEndPoint.WriteEverySeconds,
                            checkConfigSeconds = Multiplexer.RawConfig.ConfigCheckSeconds;

                        if (state == (int)State.ConnectedEstablished && ConnectionType == ConnectionType.Interactive &&
                            checkConfigSeconds > 0 && ServerEndPoint.LastInfoReplicationCheckSecondsAgo >= checkConfigSeconds &&
                            ServerEndPoint.CheckInfoReplication())
                        {
                            // that serves as a keep-alive, if it is accepted
                        }
                        else if (writeEverySeconds > 0 && tmp.LastWriteSecondsAgo >= writeEverySeconds)
                        {
                            Trace("OnHeartbeat - overdue");
                            if (state == (int)State.ConnectedEstablished)
                            {
                                KeepAlive();
                            }
                            else
                            {
                                OnDisconnected(ConnectionFailureType.SocketFailure, tmp, out bool ignore, out State oldState);
                            }
                        }
                        else if (writeEverySeconds <= 0 && tmp.IsIdle() &&
                                 tmp.LastWriteSecondsAgo > 2 &&
                                 tmp.GetSentAwaitingResponseCount() != 0)
                        {
                            // there's a chance this is a dead socket; sending data will shake that
                            // up a bit, so if we have an empty unsent queue and a non-empty sent
                            // queue, test the socket
                            KeepAlive();
                        }
                    }
                    break;

                case (int)State.Disconnected:
                    Interlocked.Exchange(ref connectTimeoutRetryCount, 0);
                    if (!ifConnectedOnly)
                    {
                        Multiplexer.Trace("Resurrecting " + ToString());
                        Multiplexer.OnResurrecting(ServerEndPoint?.EndPoint, ConnectionType);
                        GetConnection(null);
                    }
                    break;

                default:
                    Interlocked.Exchange(ref connectTimeoutRetryCount, 0);
                    break;
                }
            }
            catch (Exception ex)
            {
                OnInternalError(ex);
                Trace("OnHeartbeat error: " + ex.Message);
            }
            finally
            {
                if (runThisTime)
                {
                    Interlocked.Exchange(ref beating, 0);
                }
            }
        }
        private WriteResult WriteMessageToServerInsideWriteLock(PhysicalConnection connection, Message message)
        {
            if (message == null)
            {
                return(WriteResult.Success);                 // for some definition of success
            }
            bool isQueued = false;

            try
            {
                var cmd = message.Command;
                LastCommand = cmd;
                bool isMasterOnly = message.IsMasterOnly();

                if (isMasterOnly && ServerEndPoint.IsSlave && (ServerEndPoint.SlaveReadOnly || !ServerEndPoint.AllowSlaveWrites))
                {
                    throw ExceptionFactory.MasterOnly(Multiplexer.IncludeDetailInExceptions, message.Command, message, ServerEndPoint);
                }
                switch (cmd)
                {
                case RedisCommand.QUIT:
                    connection.RecordQuit();
                    break;

                case RedisCommand.EXEC:
                    Multiplexer.OnPreTransactionExec(message);     // testing purposes, to force certain errors
                    break;
                }

                SelectDatabaseInsideWriteLock(connection, message);

                if (!connection.TransactionActive)
                {
                    var readmode = connection.GetReadModeCommand(isMasterOnly);
                    if (readmode != null)
                    {
                        connection.EnqueueInsideWriteLock(readmode);
                        readmode.WriteTo(connection);
                        readmode.SetRequestSent();
                        IncrementOpCount();
                    }

                    if (message.IsAsking)
                    {
                        var asking = ReusableAskingCommand;
                        connection.EnqueueInsideWriteLock(asking);
                        asking.WriteTo(connection);
                        asking.SetRequestSent();
                        IncrementOpCount();
                    }
                }
                switch (cmd)
                {
                case RedisCommand.WATCH:
                case RedisCommand.MULTI:
                    connection.TransactionActive = true;
                    break;

                case RedisCommand.UNWATCH:
                case RedisCommand.EXEC:
                case RedisCommand.DISCARD:
                    connection.TransactionActive = false;
                    break;
                }

                connection.EnqueueInsideWriteLock(message);
                isQueued = true;
                message.WriteTo(connection);

                message.SetRequestSent();
                IncrementOpCount();

                // some commands smash our ability to trust the database; some commands
                // demand an immediate flush
                switch (cmd)
                {
                case RedisCommand.EVAL:
                case RedisCommand.EVALSHA:
                    if (!ServerEndPoint.GetFeatures().ScriptingDatabaseSafe)
                    {
                        connection.SetUnknownDatabase();
                    }
                    break;

                case RedisCommand.UNKNOWN:
                case RedisCommand.DISCARD:
                case RedisCommand.EXEC:
                    connection.SetUnknownDatabase();
                    break;
                }
                return(WriteResult.Success);
            }
            catch (RedisCommandException ex) when(!isQueued)
            {
                Trace("Write failed: " + ex.Message);
                message.Fail(ConnectionFailureType.InternalFailure, ex, null);
                this.CompleteSyncOrAsync(message);
                // this failed without actually writing; we're OK with that... unless there's a transaction

                if (connection?.TransactionActive == true)
                {
                    // we left it in a broken state; need to kill the connection
                    connection.RecordConnectionFailed(ConnectionFailureType.ProtocolFailure, ex);
                    return(WriteResult.WriteFailure);
                }
                return(WriteResult.Success);
            }
            catch (Exception ex)
            {
                Trace("Write failed: " + ex.Message);
                message.Fail(ConnectionFailureType.InternalFailure, ex, null);
                this.CompleteSyncOrAsync(message);

                // we're not sure *what* happened here; probably an IOException; kill the connection
                connection?.RecordConnectionFailed(ConnectionFailureType.InternalFailure, ex);
                return(WriteResult.WriteFailure);
            }
        }
 public static ProfileStorage NewWithContext(ConcurrentProfileStorageCollection pushTo, ServerEndPoint server)
 {
     return(new ProfileStorage(pushTo, server, null, null));
 }
 public static ProfileStorage NewAttachedToSameContext(ProfileStorage resentFor, ServerEndPoint server, bool isMoved)
 {
     return(new ProfileStorage(resentFor.PushToWhenFinished, server, resentFor, isMoved ? RetransmissionReasonType.Moved : RetransmissionReasonType.Ask));
 }
Esempio n. 20
0
        internal override Task <T> ExecuteAsync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
        {
            if (message == null)
            {
                return(CompletedTask <T> .Default(asyncState));
            }
            multiplexer.CheckMessage(message);

            multiplexer.Trace("Wrapping " + message.Command, "Transaction");
            // prepare the inner command as a task
            Task <T> task;

            if (message.IsFireAndForget)
            {
                task = CompletedTask <T> .Default(null); // F+F explicitly does not get async-state
            }
            else
            {
                var tcs    = TaskSource.CreateDenyExecSync <T>(asyncState);
                var source = ResultBox <T> .Get(tcs);

                message.SetSource(source, processor);
                task = tcs.Task;
            }

            // prepare an outer-command that decorates that, but expects QUEUED
            var queued    = new QueuedMessage(message);
            var wasQueued = ResultBox <bool> .Get(null);

            queued.SetSource(wasQueued, QueuedProcessor.Default);

            // store it, and return the task of the *outer* command
            // (there is no task for the inner command)
            (pending ?? (pending = new List <QueuedMessage>())).Add(queued);


            switch (message.Command)
            {
            case RedisCommand.UNKNOWN:
            case RedisCommand.EVAL:
            case RedisCommand.EVALSHA:
                // people can do very naughty things in an EVAL
                // including change the DB; change it back to what we
                // think it should be!
                var sel = PhysicalConnection.GetSelectDatabaseCommand(message.Db);
                queued    = new QueuedMessage(sel);
                wasQueued = ResultBox <bool> .Get(null);

                queued.SetSource(wasQueued, QueuedProcessor.Default);
                pending.Add(queued);
                break;
            }
            return(task);
        }
Esempio n. 21
0
        internal virtual Task <T[]> ExecuteAsync <T>(Message[] messages, ResultProcessor <T> processor, ServerEndPoint server = null)
        {
            if (messages == null)
            {
                return(CompletedTask <T[]> .Default(asyncState));
            }
            foreach (var message in messages)
            {
                multiplexer.CheckMessage(message);
            }

            return(multiplexer.ExecuteAsyncImpl <T>(messages, processor, asyncState, server));
        }
Esempio n. 22
0
        internal static Exception Timeout(ConnectionMultiplexer mutiplexer, string baseErrorMessage, Message message, ServerEndPoint server)
        {
            List <Tuple <string, string> > data = new List <Tuple <string, string> > {
                Tuple.Create("Message", message.CommandAndKey)
            };
            var sb = new StringBuilder();

            if (!string.IsNullOrEmpty(baseErrorMessage))
            {
                sb.Append(baseErrorMessage);
            }
            else
            {
                sb.Append("Timeout performing ").Append(message.CommandAndKey).Append(" (").Append(Format.ToString(mutiplexer.TimeoutMilliseconds)).Append("ms)");
            }

            void add(string lk, string sk, string v)
            {
                if (v != null)
                {
                    data.Add(Tuple.Create(lk, v));
                    sb.Append(", ").Append(sk).Append(": ").Append(v);
                }
            }

            if (server != null)
            {
                server.GetOutstandingCount(message.Command, out int inst, out int qs, out int @in);
                add("Instantaneous", "inst", inst.ToString());
                add("Queue-Awaiting-Response", "qs", qs.ToString());
                if (@in >= 0)
                {
                    add("Inbound-Bytes", "in", @in.ToString());
                }

                if (mutiplexer.StormLogThreshold >= 0 && qs >= mutiplexer.StormLogThreshold && Interlocked.CompareExchange(ref mutiplexer.haveStormLog, 1, 0) == 0)
                {
                    var log = server.GetStormLog(message.Command);
                    if (string.IsNullOrWhiteSpace(log))
                    {
                        Interlocked.Exchange(ref mutiplexer.haveStormLog, 0);
                    }
                    else
                    {
                        Interlocked.Exchange(ref mutiplexer.stormLogSnapshot, log);
                    }
                }
                add("Server-Endpoint", "serverEndpoint", server.EndPoint.ToString());
            }
            add("Manager", "mgr", mutiplexer.SocketManager?.GetState());

            add("Client-Name", "clientName", mutiplexer.ClientName);
            var hashSlot = message.GetHashSlot(mutiplexer.ServerSelectionStrategy);

            // only add keyslot if its a valid cluster key slot
            if (hashSlot != ServerSelectionStrategy.NoSlot)
            {
                add("Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(mutiplexer.ServerSelectionStrategy).ToString());
            }
            int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);

            add("ThreadPool-IO-Completion", "IOCP", iocp);
            add("ThreadPool-Workers", "WORKER", worker);
            data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));

            if (mutiplexer.IncludePerformanceCountersInExceptions)
            {
                add("Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
            }

            add("Version", "v", GetLibVersion());

            sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
            sb.Append(timeoutHelpLink);
            sb.Append(")");

            var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
            {
                HelpLink = timeoutHelpLink
            };

            if (data != null)
            {
                foreach (var kv in data)
                {
                    ex.Data["Redis-" + kv.Item1] = kv.Item2;
                }
            }

            if (mutiplexer.IncludeDetailInExceptions)
            {
                AddDetail(ex, message, server, null);
            }
            return(ex);
        }
Esempio n. 23
0
        internal static Exception NoConnectionAvailable(
            ConnectionMultiplexer multiplexer,
            Message message,
            ServerEndPoint server,
            ReadOnlySpan <ServerEndPoint> serverSnapshot = default,
            RedisCommand command = default)
        {
            string commandLabel = GetLabel(multiplexer.IncludeDetailInExceptions, message?.Command ?? command, message);

            if (server != null)
            {
                //if we already have the serverEndpoint for connection failure use that
                //otherwise it would output state of all the endpoints
                serverSnapshot = new ServerEndPoint[] { server };
            }

            var innerException = PopulateInnerExceptions(serverSnapshot == default ? multiplexer.GetServerSnapshot() : serverSnapshot);

            // Try to get a useful error message for the user.
            long   attempts = multiplexer._connectAttemptCount, completions = multiplexer._connectCompletedCount;
            string initialMessage;

            // We only need to customize the connection if we're aborting on connect fail
            // The "never" case would have thrown, if this was true
            if (!multiplexer.RawConfig.AbortOnConnectFail && attempts <= multiplexer.RawConfig.ConnectRetry && completions == 0)
            {
                // Initial attempt, attempted use before an async connection completes
                initialMessage = $"Connection to Redis never succeeded (attempts: {attempts} - connection likely in-progress), unable to service operation: ";
            }
            else if (!multiplexer.RawConfig.AbortOnConnectFail && attempts > multiplexer.RawConfig.ConnectRetry && completions == 0)
            {
                // Attempted use after a full initial retry connect count # of failures
                // This can happen in Azure often, where user disables abort and has the wrong config
                initialMessage = $"Connection to Redis never succeeded (attempts: {attempts} - check your config), unable to service operation: ";
            }
            else
            {
                // Default if we don't have a more useful error message here based on circumstances
                initialMessage = "No connection is active/available to service this operation: ";
            }

            StringBuilder sb = new StringBuilder(initialMessage);

            sb.Append(commandLabel);
            string innermostExceptionstring = GetInnerMostExceptionMessage(innerException);

            if (!string.IsNullOrEmpty(innermostExceptionstring))
            {
                sb.Append("; ").Append(innermostExceptionstring);
            }

            // Add counters and exception data if we have it
            List <Tuple <string, string> > data = null;

            if (multiplexer.IncludeDetailInExceptions)
            {
                data = new List <Tuple <string, string> >();
                AddCommonDetail(data, sb, message, multiplexer, server);
            }
            var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, sb.ToString(), innerException, message?.Status ?? CommandStatus.Unknown);

            if (multiplexer.IncludeDetailInExceptions)
            {
                CopyDataToException(data, ex);
                sb.Append("; ").Append(PerfCounterHelper.GetThreadPoolAndCPUSummary(multiplexer.IncludePerformanceCountersInExceptions));
                AddExceptionDetail(ex, message, server, commandLabel);
            }
            return(ex);
        }
Esempio n. 24
0
 internal RedisServer(ConnectionMultiplexer multiplexer, ServerEndPoint server, object asyncState) : base(multiplexer, asyncState)
 {
     this.server = server ?? throw new ArgumentNullException(nameof(server));
 }
        internal static Exception Timeout(bool includeDetail, string errorMessage, Message message, ServerEndPoint server)
        {
            var ex = new TimeoutException(errorMessage);

            if (includeDetail)
            {
                AddDetail(ex, message, server, null);
            }
            return(ex);
        }
        internal static Exception Timeout(bool includeDetail, string errorMessage, Message message, ServerEndPoint server)
        {
            var ex = new RedisTimeoutException(errorMessage, message?.Status ?? CommandStatus.Unknown);

            if (includeDetail)
            {
                AddDetail(ex, message, server, null);
            }
            return(ex);
        }
        internal static Exception ConnectionFailure(bool includeDetail, ConnectionFailureType failureType, string message, ServerEndPoint server)
        {
            var ex = new RedisConnectionException(failureType, message);

            if (includeDetail)
            {
                AddDetail(ex, null, server, null);
            }
            return(ex);
        }
        internal static Exception TooManyArgs(bool includeDetail, string command, Message message, ServerEndPoint server, int required)
        {
            string s  = GetLabel(includeDetail, command, message);
            var    ex = new RedisCommandException($"This operation would involve too many arguments ({required} vs the redis limit of {PhysicalConnection.REDIS_MAX_ARGS}): {s}");

            if (includeDetail)
            {
                AddDetail(ex, message, server, s);
            }
            return(ex);
        }
        internal static Exception MasterOnly(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
        {
            string s  = GetLabel(includeDetail, command, message);
            var    ex = new RedisCommandException("Command cannot be issued to a slave: " + s);

            if (includeDetail)
            {
                AddDetail(ex, message, server, s);
            }
            return(ex);
        }
Esempio n. 30
0
 internal override Task <T> ExecuteAsync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
 {   // inject our expected server automatically
     if (server == null)
     {
         server = this.server;
     }
     FixFlags(message, server);
     if (!server.IsConnected)
     {
         if (message == null)
         {
             return(CompletedTask <T> .Default(asyncState));
         }
         if (message.IsFireAndForget)
         {
             return(CompletedTask <T> .Default(null));                       // F+F explicitly does not get async-state
         }
         // no need to deny exec-sync here; will be complete before they see if
         var tcs = TaskSource.Create <T>(asyncState);
         ConnectionMultiplexer.ThrowFailed(tcs, ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server));
         return(tcs.Task);
     }
     return(base.ExecuteAsync <T>(message, processor, server));
 }
Esempio n. 31
0
        internal override Task <T> ExecuteAsync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
        {
            if (message == null)
            {
                return(CompletedTask <T> .Default(asyncState));
            }
            multiplexer.CheckMessage(message);

            // prepare the inner command as a task
            Task <T> task;

            if (message.IsFireAndForget)
            {
                task = CompletedTask <T> .Default(null); // F+F explicitly does not get async-state
            }
            else
            {
                var source = TaskResultBox <T> .Create(out var tcs, asyncState);

                task = tcs.Task;
                message.SetSource(source, processor);
            }

            // store it
            (pending ??= new List <Message>()).Add(message);
            return(task);
        }
Esempio n. 32
0
 internal override T ExecuteSync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
 {   // inject our expected server automatically
     if (server == null)
     {
         server = this.server;
     }
     FixFlags(message, server);
     if (!server.IsConnected)
     {
         if (message == null || message.IsFireAndForget)
         {
             return(default(T));
         }
         throw ExceptionFactory.NoConnectionAvailable(multiplexer.IncludeDetailInExceptions, message.Command, message, server);
     }
     return(base.ExecuteSync <T>(message, processor, server));
 }
 public static ProfileStorage NewWithContext(ConcurrentProfileStorageCollection pushTo, ServerEndPoint server)
 {
     return new ProfileStorage(pushTo, server, null, null);
 }
Esempio n. 34
0
 internal override RedisFeatures GetFeatures(int db, RedisKey key, CommandFlags flags, out ServerEndPoint server)
 {
     server = this.server;
     return(new RedisFeatures(server.Version));
 }
Esempio n. 35
0
        internal virtual RedisFeatures GetFeatures(int db, RedisKey key, CommandFlags flags, out ServerEndPoint server)
        {
            server = multiplexer.SelectServer(db, RedisCommand.PING, flags, key);
            var version = server == null ? multiplexer.RawConfig.DefaultVersion : server.Version;

            return(new RedisFeatures(version));
        }
        internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server, ServerEndPoint[] serverSnapshot)
        {
            string commandLabel = GetLabel(includeDetail, command, message);

            if (server != null)
            {
                //if we already have the serverEndpoint for connection failure use that
                //otherwise it would output state of all the endpoints
                serverSnapshot = new ServerEndPoint[] { server };
            }

            var innerException = PopulateInnerExceptions(serverSnapshot);

            StringBuilder exceptionmessage         = new StringBuilder("No connection is available to service this operation: ").Append(commandLabel);
            string        innermostExceptionstring = GetInnerMostExceptionMessage(innerException);

            if (!string.IsNullOrEmpty(innermostExceptionstring))
            {
                exceptionmessage.Append("; ").Append(innermostExceptionstring);
            }

#if !CORE_CLR
            if (includeDetail)
            {
                exceptionmessage.Append("; ").Append(ConnectionMultiplexer.GetThreadPoolAndCPUSummary());
            }
#endif

            var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, exceptionmessage.ToString(), innerException);

            if (includeDetail)
            {
                AddDetail(ex, message, server, commandLabel);
            }
            return(ex);
        }
 internal static Exception DatabaseOutfRange(bool includeDetail, int targetDatabase, Message message, ServerEndPoint server)
 {
     var ex = new RedisCommandException("The database does not exist on the server: " + targetDatabase);
     if (includeDetail) AddDetail(ex, message, server, null);
     return ex;
 }
        internal static Exception DatabaseOutfRange(bool includeDetail, int targetDatabase, Message message, ServerEndPoint server)
        {
            var ex = new RedisCommandException("The database does not exist on the server: " + targetDatabase);

            if (includeDetail)
            {
                AddDetail(ex, message, server, null);
            }
            return(ex);
        }
        internal static Exception AdminModeNotEnabled(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
        {
            string s  = GetLabel(includeDetail, command, message);
            var    ex = new RedisCommandException("This operation is not available unless admin mode is enabled: " + s);

            if (includeDetail)
            {
                AddDetail(ex, message, server, s);
            }
            return(ex);
        }
        private ServerEndPoint FindSlave(ServerEndPoint endpoint, RedisCommand command)
        {
            if (endpoint.IsSlave && endpoint.IsSelectable(command)) return endpoint;

            var slaves = endpoint.Slaves;
            for (int i = 0; i < slaves.Length; i++)
            {
                endpoint = slaves[i];
                if (endpoint.IsSlave && endpoint.IsSelectable(command)) return endpoint;
            }
            return null;
        }
Esempio n. 41
0
 internal virtual Task <T> ExecuteAsync <T>(Message message, ResultProcessor <T> processor, ServerEndPoint server = null)
 {
     if (message == null)
     {
         return(CompletedTask <T> .Default(asyncState));
     }
     multiplexer.CheckMessage(message);
     return(multiplexer.ExecuteAsyncImpl <T>(message, processor, asyncState, server));
 }
 internal void UpdateClusterRange(int fromInclusive, int toInclusive, ServerEndPoint server)
 {
     var arr = MapForMutation();
     for (int i = fromInclusive; i <= toInclusive; i++)
     {
         arr[i] = server;
     }
 }
        private ServerEndPoint FindMaster(ServerEndPoint endpoint, RedisCommand command)
        {
            int max = 5;
            do
            {
                if (!endpoint.IsSlave && endpoint.IsSelectable(command)) return endpoint;

                endpoint = endpoint.Master;
            } while (endpoint != null && --max != 0);
            return null;
        }
Esempio n. 44
0
        internal static Exception Timeout(ConnectionMultiplexer multiplexer, string baseErrorMessage, Message message, ServerEndPoint server, WriteResult?result = null)
        {
            List <Tuple <string, string> > data = new List <Tuple <string, string> > {
                Tuple.Create("Message", message.CommandAndKey)
            };
            var sb = new StringBuilder();

            if (!string.IsNullOrEmpty(baseErrorMessage))
            {
                sb.Append(baseErrorMessage);
                if (message != null)
                {
                    sb.Append(", command=").Append(message.Command); // no key here, note
                }
            }
            else
            {
                sb.Append("Timeout performing ").Append(message.Command).Append(" (").Append(Format.ToString(multiplexer.TimeoutMilliseconds)).Append("ms)");
            }

            // Add timeout data, if we have it
            if (result == WriteResult.TimeoutBeforeWrite)
            {
                Add(data, sb, "Timeout", "timeout", Format.ToString(multiplexer.TimeoutMilliseconds));
                try
                {
#if DEBUG
                    if (message.QueuePosition >= 0)
                    {
                        Add(data, sb, "QueuePosition", null, message.QueuePosition.ToString());                             // the position the item was when added to the queue
                    }
                    if ((int)message.ConnectionWriteState >= 0)
                    {
                        Add(data, sb, "WriteState", null, message.ConnectionWriteState.ToString());                                         // what the physical was doing when it was added to the queue
                    }
#endif
                    if (message != null && message.TryGetPhysicalState(out var ws, out var rs, out var sentDelta, out var receivedDelta))
                    {
                        Add(data, sb, "Write-State", null, ws.ToString());
                        Add(data, sb, "Read-State", null, rs.ToString());
                        // these might not always be available
                        if (sentDelta >= 0)
                        {
                            Add(data, sb, "OutboundDeltaKB", "outbound", $"{sentDelta >> 10}KiB");
                        }
                        if (receivedDelta >= 0)
                        {
                            Add(data, sb, "InboundDeltaKB", "inbound", $"{receivedDelta >> 10}KiB");
                        }
                    }
                }
                catch { }
            }

            AddCommonDetail(data, sb, message, multiplexer, server);

            sb.Append(" (Please take a look at this article for some common client-side issues that can cause timeouts: ");
            sb.Append(timeoutHelpLink);
            sb.Append(')');

            var ex = new RedisTimeoutException(sb.ToString(), message?.Status ?? CommandStatus.Unknown)
            {
                HelpLink = timeoutHelpLink
            };
            CopyDataToException(data, ex);

            if (multiplexer.IncludeDetailInExceptions)
            {
                AddExceptionDetail(ex, message, server, null);
            }
            return(ex);
        }
        private static void AddDetail(Exception exception, Message message, ServerEndPoint server, string label)
        {
            if (exception != null)
            {
                if (message != null) exception.Data.Add(DataCommandKey, message.CommandAndKey);
                else if (label != null) exception.Data.Add(DataCommandKey, label);

                if (server != null) exception.Data.Add(DataServerKey, Format.ToString(server.EndPoint));
            }
        }
Esempio n. 46
0
        private static void AddCommonDetail(
            List <Tuple <string, string> > data,
            StringBuilder sb,
            Message message,
            ConnectionMultiplexer multiplexer,
            ServerEndPoint server
            )
        {
            if (message != null)
            {
                message.TryGetHeadMessages(out var now, out var next);
                if (now != null)
                {
                    Add(data, sb, "Message-Current", "active", multiplexer.IncludeDetailInExceptions ? now.CommandAndKey : now.Command.ToString());
                }
                if (next != null)
                {
                    Add(data, sb, "Message-Next", "next", multiplexer.IncludeDetailInExceptions ? next.CommandAndKey : next.Command.ToString());
                }
            }

            // Add server data, if we have it
            if (server != null && message != null)
            {
                server.GetOutstandingCount(message.Command, out int inst, out int qs, out long @in, out int qu, out bool aw, out long toRead, out long toWrite, out var bs, out var rs, out var ws);
                switch (rs)
                {
                case PhysicalConnection.ReadStatus.CompletePendingMessageAsync:
                case PhysicalConnection.ReadStatus.CompletePendingMessageSync:
                    sb.Append(" ** possible thread-theft indicated; see https://stackexchange.github.io/StackExchange.Redis/ThreadTheft ** ");
                    break;
                }
                Add(data, sb, "OpsSinceLastHeartbeat", "inst", inst.ToString());
                Add(data, sb, "Queue-Awaiting-Write", "qu", qu.ToString());
                Add(data, sb, "Queue-Awaiting-Response", "qs", qs.ToString());
                Add(data, sb, "Active-Writer", "aw", aw.ToString());
                if (qu != 0)
                {
                    Add(data, sb, "Backlog-Writer", "bw", bs.ToString());
                }
                if (rs != PhysicalConnection.ReadStatus.NA)
                {
                    Add(data, sb, "Read-State", "rs", rs.ToString());
                }
                if (ws != PhysicalConnection.WriteStatus.NA)
                {
                    Add(data, sb, "Write-State", "ws", ws.ToString());
                }

                if (@in >= 0)
                {
                    Add(data, sb, "Inbound-Bytes", "in", @in.ToString());
                }
                if (toRead >= 0)
                {
                    Add(data, sb, "Inbound-Pipe-Bytes", "in-pipe", toRead.ToString());
                }
                if (toWrite >= 0)
                {
                    Add(data, sb, "Outbound-Pipe-Bytes", "out-pipe", toWrite.ToString());
                }

                if (multiplexer.StormLogThreshold >= 0 && qs >= multiplexer.StormLogThreshold && Interlocked.CompareExchange(ref multiplexer.haveStormLog, 1, 0) == 0)
                {
                    var log = server.GetStormLog(message.Command);
                    if (string.IsNullOrWhiteSpace(log))
                    {
                        Interlocked.Exchange(ref multiplexer.haveStormLog, 0);
                    }
                    else
                    {
                        Interlocked.Exchange(ref multiplexer.stormLogSnapshot, log);
                    }
                }
                Add(data, sb, "Server-Endpoint", "serverEndpoint", server.EndPoint.ToString().Replace("Unspecified/", ""));
            }
            Add(data, sb, "Multiplexer-Connects", "mc", $"{multiplexer._connectAttemptCount}/{multiplexer._connectCompletedCount}/{multiplexer._connectionCloseCount}");
            Add(data, sb, "Manager", "mgr", multiplexer.SocketManager?.GetState());

            Add(data, sb, "Client-Name", "clientName", multiplexer.ClientName);
            if (message != null)
            {
                var hashSlot = message.GetHashSlot(multiplexer.ServerSelectionStrategy);
                // only add keyslot if its a valid cluster key slot
                if (hashSlot != ServerSelectionStrategy.NoSlot)
                {
                    Add(data, sb, "Key-HashSlot", "PerfCounterHelperkeyHashSlot", message.GetHashSlot(multiplexer.ServerSelectionStrategy).ToString());
                }
            }
            int busyWorkerCount = PerfCounterHelper.GetThreadPoolStats(out string iocp, out string worker);

            Add(data, sb, "ThreadPool-IO-Completion", "IOCP", iocp);
            Add(data, sb, "ThreadPool-Workers", "WORKER", worker);
            data.Add(Tuple.Create("Busy-Workers", busyWorkerCount.ToString()));

            if (multiplexer.IncludePerformanceCountersInExceptions)
            {
                Add(data, sb, "Local-CPU", "Local-CPU", PerfCounterHelper.GetSystemCpuPercent());
            }

            Add(data, sb, "Version", "v", GetLibVersion());
        }
 internal static Exception Timeout(bool includeDetail, string errorMessage, Message message, ServerEndPoint server)
 {
     var ex = new TimeoutException(errorMessage);
     if (includeDetail) AddDetail(ex, message, server, null);
     return ex;
 }
 internal static Exception NoConnectionAvailable(bool includeDetail, RedisCommand command, Message message, ServerEndPoint server)
 {
     string s = GetLabel(includeDetail, command, message);
     var ex = new RedisConnectionException(ConnectionFailureType.UnableToResolvePhysicalConnection, "No connection is available to service this operation: " + s);
     if (includeDetail) AddDetail(ex, message, server, s);
     return ex;
 }