private void AlreadyCompleted(int status)
        {
            // Only need to release resources from AsyncTimeoutQueue timeout.
            // Otherwise, resources have already been released.
            if (status == FAIL_TOTAL_TIMEOUT)
            {
                // Free up resources and notify user on timeout.
                // Connection should have already been closed on AsyncTimeoutQueue timeout.
                PutBackArgsOnError();
                NotifyFailure(new AerospikeException.Timeout(node, policy, iteration + 1, true));
            }
            else if (status == FAIL_SOCKET_TIMEOUT)
            {
                // Connection should have already been closed on AsyncTimeoutQueue timeout.
                AerospikeException timeoutException = new AerospikeException.Timeout(node, policy, iteration + 1, true);

                if (iteration < policy.maxRetries)
                {
                    if (isRead)
                    {
                        // Read commands shift to prole node on socket timeout.
                        sequence++;
                    }
                    Retry(timeoutException);
                }
                else
                {
                    PutBackArgsOnError();
                    NotifyFailure(timeoutException);
                }
            }
            else
            {
                if (Log.WarnEnabled())
                {
                    Log.Warn("AsyncCommand unexpected return status: " + status);
                }
            }
        }
예제 #2
0
            private void HandleSocketEvent(object sender, SocketAsyncEventArgs args)
            {
                AsyncCommand command = args.UserToken as AsyncCommand;

                if (args.SocketError != SocketError.Success)
                {
                    command.ConnectionFailed(command.GetAerospikeException(args.SocketError));
                    return;
                }

                try
                {
                    switch (args.LastOperation)
                    {
                    case SocketAsyncOperation.Receive:
                        command.ReceiveEvent();
                        break;

                    case SocketAsyncOperation.Send:
                        command.SendEvent();
                        break;

                    case SocketAsyncOperation.Connect:
                        command.ConnectionCreated();
                        break;

                    default:
                        command.FailOnApplicationError(new AerospikeException("Invalid socket operation: " + args.LastOperation));
                        break;
                    }
                }
                catch (AerospikeException.Connection ac)
                {
                    command.ConnectionFailed(ac);
                }
                catch (AerospikeException ae)
                {
                    // Fail without retry on non-network errors.
                    if (ae.Result == ResultCode.TIMEOUT)
                    {
                        // Create server timeout exception.
                        ae = new AerospikeException.Timeout(command.policy, false);
                    }
                    command.FailOnApplicationError(ae);
                }
                catch (SocketException se)
                {
                    command.ConnectionFailed(command.GetAerospikeException(se.SocketErrorCode));
                }
                catch (ObjectDisposedException ode)
                {
                    // This exception occurs because socket is being used after timeout thread closes socket.
                    // Retry when this happens.
                    command.ConnectionFailed(new AerospikeException(ode));
                }
                catch (Exception e)
                {
                    // Fail without retry on unknown errors.
                    command.FailOnApplicationError(new AerospikeException(e));
                }
            }
예제 #3
0
        public void ExecuteCommand()
        {
            Node node;
            AerospikeException exception = null;
            bool isClientTimeout;

            // Execute command until successful, timed out or maximum iterations have been reached.
            while (true)
            {
                try
                {
                    node = GetNode();
                }
                catch (AerospikeException ae)
                {
                    ae.Policy    = policy;
                    ae.Iteration = iteration;
                    ae.SetInDoubt(IsWrite(), commandSentCounter);
                    throw;
                }

                try
                {
                    Connection conn = node.GetConnection(socketTimeout);

                    try
                    {
                        // Set command buffer.
                        WriteBuffer();

                        // Send command.
                        conn.Write(dataBuffer, dataOffset);
                        commandSentCounter++;

                        // Parse results.
                        ParseResult(conn);

                        // Put connection back in pool.
                        node.PutConnection(conn);

                        // Command has completed successfully.  Exit method.
                        return;
                    }
                    catch (AerospikeException ae)
                    {
                        if (ae.KeepConnection())
                        {
                            // Put connection back in pool.
                            node.PutConnection(conn);
                        }
                        else
                        {
                            // Close socket to flush out possible garbage.  Do not put back in pool.
                            node.CloseConnection(conn);
                        }

                        if (ae.Result == ResultCode.TIMEOUT)
                        {
                            // Go through retry logic on server timeout.
                            exception       = new AerospikeException.Timeout(policy, false);
                            isClientTimeout = false;
                        }
                        else
                        {
                            throw;
                        }
                    }
                    catch (SocketException se)
                    {
                        // Socket errors are considered temporary anomalies.
                        // Retry after closing connection.
                        node.CloseConnection(conn);

                        if (se.SocketErrorCode == SocketError.TimedOut)
                        {
                            isClientTimeout = true;
                        }
                        else
                        {
                            exception       = new AerospikeException.Connection(se);
                            isClientTimeout = false;
                        }
                    }
                    catch (Exception)
                    {
                        // All other exceptions are considered fatal.  Do not retry.
                        // Close socket to flush out possible garbage.  Do not put back in pool.
                        node.CloseConnection(conn);
                        throw;
                    }
                }
                catch (SocketException se)
                {
                    // This exception might happen after initial connection succeeded, but
                    // user login failed with a socket error.  Retry.
                    if (se.SocketErrorCode == SocketError.TimedOut)
                    {
                        isClientTimeout = true;
                    }
                    else
                    {
                        exception       = new AerospikeException.Connection(se);
                        isClientTimeout = false;
                    }
                }
                catch (AerospikeException.Connection ce)
                {
                    // Socket connection error has occurred. Retry.
                    exception       = ce;
                    isClientTimeout = false;
                }
                catch (AerospikeException ae)
                {
                    ae.Node      = node;
                    ae.Policy    = policy;
                    ae.Iteration = iteration;
                    ae.SetInDoubt(IsWrite(), commandSentCounter);
                    throw;
                }

                // Check maxRetries.
                if (iteration > maxRetries)
                {
                    break;
                }

                if (totalTimeout > 0)
                {
                    // Check for total timeout.
                    long remaining = (long)deadline.Subtract(DateTime.UtcNow).TotalMilliseconds - policy.sleepBetweenRetries;

                    if (remaining <= 0)
                    {
                        break;
                    }

                    if (remaining < totalTimeout)
                    {
                        totalTimeout = (int)remaining;

                        if (socketTimeout > totalTimeout)
                        {
                            socketTimeout = totalTimeout;
                        }
                    }
                }

                if (!isClientTimeout && policy.sleepBetweenRetries > 0)
                {
                    // Sleep before trying again.
                    Util.Sleep(policy.sleepBetweenRetries);
                }

                iteration++;

                if (!PrepareRetry(isClientTimeout || exception.Result != ResultCode.SERVER_NOT_AVAILABLE))
                {
                    // Batch may be retried in separate commands.
                    if (RetryBatch(cluster, socketTimeout, totalTimeout, deadline, iteration, commandSentCounter))
                    {
                        // Batch was retried in separate commands.  Complete this command.
                        return;
                    }
                }
            }

            // Retries have been exhausted.  Throw last exception.
            if (isClientTimeout)
            {
                exception = new AerospikeException.Timeout(policy, true);
            }
            exception.Node      = node;
            exception.Policy    = policy;
            exception.Iteration = iteration;
            exception.SetInDoubt(IsWrite(), commandSentCounter);
            throw exception;
        }
예제 #4
0
        public void Execute(Cluster cluster, Policy policy, Key key, Node node, bool isRead)
        {
            Partition          partition = (key != null)? new Partition(key) : null;
            AerospikeException exception = null;
            DateTime           deadline  = DateTime.MinValue;
            int  socketTimeout           = policy.socketTimeout;
            int  totalTimeout            = policy.totalTimeout;
            int  iteration          = 0;
            int  commandSentCounter = 0;
            bool isClientTimeout;

            if (totalTimeout > 0)
            {
                deadline = DateTime.UtcNow.AddMilliseconds(totalTimeout);

                if (socketTimeout == 0 || socketTimeout > totalTimeout)
                {
                    socketTimeout = totalTimeout;
                }
            }

            // Execute command until successful, timed out or maximum iterations have been reached.
            while (true)
            {
                try
                {
                    if (partition != null)
                    {
                        // Single record command node retrieval.
                        node = GetNode(cluster, partition, policy.replica, isRead);
                    }

                    Connection conn = node.GetConnection(socketTimeout);

                    try
                    {
                        // Set command buffer.
                        WriteBuffer();

                        // Check if timeout needs to be changed in send buffer.
                        if (totalTimeout != policy.totalTimeout)
                        {
                            // Reset timeout in send buffer (destined for server) and socket.
                            ByteUtil.IntToBytes((uint)totalTimeout, dataBuffer, 22);
                        }

                        // Send command.
                        conn.Write(dataBuffer, dataOffset);
                        commandSentCounter++;

                        // Parse results.
                        ParseResult(conn);

                        // Put connection back in pool.
                        node.PutConnection(conn);

                        // Command has completed successfully.  Exit method.
                        return;
                    }
                    catch (AerospikeException ae)
                    {
                        if (ae.KeepConnection())
                        {
                            // Put connection back in pool.
                            node.PutConnection(conn);
                        }
                        else
                        {
                            // Close socket to flush out possible garbage.  Do not put back in pool.
                            node.CloseConnection(conn);
                        }

                        if (ae.Result == ResultCode.TIMEOUT)
                        {
                            // Go through retry logic on server timeout.
                            exception       = new AerospikeException.Timeout(node, policy, iteration + 1, false);
                            isClientTimeout = false;

                            if (isRead)
                            {
                                base.sequence++;
                            }
                        }
                        else
                        {
                            ae.SetInDoubt(isRead, commandSentCounter);
                            throw;
                        }
                    }
                    catch (SocketException se)
                    {
                        // Socket errors are considered temporary anomalies.
                        // Retry after closing connection.
                        node.CloseConnection(conn);

                        if (se.SocketErrorCode == SocketError.TimedOut)
                        {
                            isClientTimeout = true;

                            if (isRead)
                            {
                                base.sequence++;
                            }
                        }
                        else
                        {
                            exception       = new AerospikeException(se);
                            isClientTimeout = false;
                            base.sequence++;
                        }
                    }
                    catch (Exception)
                    {
                        // All other exceptions are considered fatal.  Do not retry.
                        // Close socket to flush out possible garbage.  Do not put back in pool.
                        node.CloseConnection(conn);
                        throw;
                    }
                }
                catch (AerospikeException.Connection ce)
                {
                    // Socket connection error has occurred. Retry.
                    exception       = ce;
                    isClientTimeout = false;
                    base.sequence++;
                }

                // Check maxRetries.
                if (++iteration > policy.maxRetries)
                {
                    break;
                }

                if (policy.totalTimeout > 0)
                {
                    // Check for total timeout.
                    long remaining = (long)deadline.Subtract(DateTime.UtcNow).TotalMilliseconds - policy.sleepBetweenRetries;

                    if (remaining <= 0)
                    {
                        break;
                    }

                    if (remaining < totalTimeout)
                    {
                        totalTimeout = (int)remaining;

                        if (socketTimeout > totalTimeout)
                        {
                            socketTimeout = totalTimeout;
                        }
                    }
                }

                if (!isClientTimeout && policy.sleepBetweenRetries > 0)
                {
                    // Sleep before trying again.
                    Util.Sleep(policy.sleepBetweenRetries);
                }
            }

            // Retries have been exhausted.  Throw last exception.
            if (isClientTimeout)
            {
                exception = new AerospikeException.Timeout(node, policy, iteration, true);
            }
            exception.SetInDoubt(isRead, commandSentCounter);
            throw exception;
        }
        public void Execute
        (
            Cluster cluster, Policy policy, Partition partition, Node node, bool isRead,
            int socketTimeout, int totalTimeout, DateTime deadline, int iteration, int commandSentCounter
        )
        {
            AerospikeException exception = null;
            bool isClientTimeout;

            // Execute command until successful, timed out or maximum iterations have been reached.
            while (true)
            {
                if (partition != null)
                {
                    // Single record command node retrieval.
                    try
                    {
                        node = GetNode(cluster, policy, partition, isRead);
                    }
                    catch (AerospikeException ae)
                    {
                        ae.Iteration = iteration;
                        ae.SetInDoubt(isRead, commandSentCounter);
                        throw;
                    }
                }

                try
                {
                    Connection conn = node.GetConnection(socketTimeout);

                    try
                    {
                        // Set command buffer.
                        WriteBuffer();

                        // Check if timeout needs to be changed in send buffer.
                        if (totalTimeout != policy.totalTimeout)
                        {
                            // Reset timeout in send buffer (destined for server) and socket.
                            ByteUtil.IntToBytes((uint)totalTimeout, dataBuffer, 22);
                        }

                        // Send command.
                        conn.Write(dataBuffer, dataOffset);
                        commandSentCounter++;

                        // Parse results.
                        ParseResult(conn);

                        // Put connection back in pool.
                        node.PutConnection(conn);

                        // Command has completed successfully.  Exit method.
                        return;
                    }
                    catch (AerospikeException ae)
                    {
                        if (ae.KeepConnection())
                        {
                            // Put connection back in pool.
                            node.PutConnection(conn);
                        }
                        else
                        {
                            // Close socket to flush out possible garbage.  Do not put back in pool.
                            node.CloseConnection(conn);
                        }

                        if (ae.Result == ResultCode.TIMEOUT)
                        {
                            // Go through retry logic on server timeout.
                            exception       = new AerospikeException.Timeout(policy, false);
                            isClientTimeout = false;
                            ShiftSequenceOnRead(policy, isRead);
                        }
                        else
                        {
                            throw;
                        }
                    }
                    catch (SocketException se)
                    {
                        // Socket errors are considered temporary anomalies.
                        // Retry after closing connection.
                        node.CloseConnection(conn);

                        if (se.SocketErrorCode == SocketError.TimedOut)
                        {
                            isClientTimeout = true;
                            ShiftSequenceOnRead(policy, isRead);
                        }
                        else
                        {
                            exception       = new AerospikeException(se);
                            isClientTimeout = false;
                            base.sequence++;
                        }
                    }
                    catch (Exception)
                    {
                        // All other exceptions are considered fatal.  Do not retry.
                        // Close socket to flush out possible garbage.  Do not put back in pool.
                        node.CloseConnection(conn);
                        throw;
                    }
                }
                catch (SocketException se)
                {
                    // This exception might happen after initial connection succeeded, but
                    // user login failed with a socket error.  Retry.
                    if (se.SocketErrorCode == SocketError.TimedOut)
                    {
                        isClientTimeout = true;
                        ShiftSequenceOnRead(policy, isRead);
                    }
                    else
                    {
                        exception       = new AerospikeException(se);
                        isClientTimeout = false;
                        base.sequence++;
                    }
                }
                catch (AerospikeException.Connection ce)
                {
                    // Socket connection error has occurred. Retry.
                    exception       = ce;
                    isClientTimeout = false;
                    base.sequence++;
                }
                catch (AerospikeException ae)
                {
                    ae.Node      = node;
                    ae.Iteration = iteration;
                    ae.SetInDoubt(isRead, commandSentCounter);
                    throw;
                }

                // Check maxRetries.
                if (iteration > policy.maxRetries)
                {
                    break;
                }

                if (policy.totalTimeout > 0)
                {
                    // Check for total timeout.
                    long remaining = (long)deadline.Subtract(DateTime.UtcNow).TotalMilliseconds - policy.sleepBetweenRetries;

                    if (remaining <= 0)
                    {
                        break;
                    }

                    if (remaining < totalTimeout)
                    {
                        totalTimeout = (int)remaining;

                        if (socketTimeout > totalTimeout)
                        {
                            socketTimeout = totalTimeout;
                        }
                    }
                }

                if (!isClientTimeout && policy.sleepBetweenRetries > 0)
                {
                    // Sleep before trying again.
                    Util.Sleep(policy.sleepBetweenRetries);
                }

                iteration++;

                if (ShouldRetryBatch() && RetryBatch(cluster, socketTimeout, totalTimeout, deadline, iteration, commandSentCounter))
                {
                    // Batch retried in separate commands.  Complete this command.
                    return;
                }
            }

            // Retries have been exhausted.  Throw last exception.
            if (isClientTimeout)
            {
                exception = new AerospikeException.Timeout(policy, true);
            }
            exception.Node      = node;
            exception.Iteration = iteration;
            exception.SetInDoubt(isRead, commandSentCounter);
            throw exception;
        }