private void AlreadyCompleted(int status)
{
// Only need to release resources from AsyncTimeoutQueue timeout.
// Otherwise, resources have already been released.
if (status == FAIL_TOTAL_TIMEOUT)
{
// Free up resources and notify user on timeout.
// Connection should have already been closed on AsyncTimeoutQueue timeout.
PutBackArgsOnError();
NotifyFailure(new AerospikeException.Timeout(node, policy, iteration + 1, true));
}
else if (status == FAIL_SOCKET_TIMEOUT)
{
// Connection should have already been closed on AsyncTimeoutQueue timeout.
AerospikeException timeoutException = new AerospikeException.Timeout(node, policy, iteration + 1, true);
if (iteration < policy.maxRetries)
{
if (isRead)
{
// Read commands shift to prole node on socket timeout.
sequence++;
}
Retry(timeoutException);
}
else
{
PutBackArgsOnError();
NotifyFailure(timeoutException);
}
}
else
{
if (Log.WarnEnabled())
{
Log.Warn("AsyncCommand unexpected return status: " + status);
}
}
}
private void HandleSocketEvent(object sender, SocketAsyncEventArgs args)
{
AsyncCommand command = args.UserToken as AsyncCommand;
if (args.SocketError != SocketError.Success)
{
command.ConnectionFailed(command.GetAerospikeException(args.SocketError));
return;
}
try
{
switch (args.LastOperation)
{
case SocketAsyncOperation.Receive:
command.ReceiveEvent();
break;
case SocketAsyncOperation.Send:
command.SendEvent();
break;
case SocketAsyncOperation.Connect:
command.ConnectionCreated();
break;
default:
command.FailOnApplicationError(new AerospikeException("Invalid socket operation: " + args.LastOperation));
break;
}
}
catch (AerospikeException.Connection ac)
{
command.ConnectionFailed(ac);
}
catch (AerospikeException ae)
{
// Fail without retry on non-network errors.
if (ae.Result == ResultCode.TIMEOUT)
{
// Create server timeout exception.
ae = new AerospikeException.Timeout(command.policy, false);
}
command.FailOnApplicationError(ae);
}
catch (SocketException se)
{
command.ConnectionFailed(command.GetAerospikeException(se.SocketErrorCode));
}
catch (ObjectDisposedException ode)
{
// This exception occurs because socket is being used after timeout thread closes socket.
// Retry when this happens.
command.ConnectionFailed(new AerospikeException(ode));
}
catch (Exception e)
{
// Fail without retry on unknown errors.
command.FailOnApplicationError(new AerospikeException(e));
}
}
public void ExecuteCommand()
{
Node node;
AerospikeException exception = null;
bool isClientTimeout;
// Execute command until successful, timed out or maximum iterations have been reached.
while (true)
{
try
{
node = GetNode();
}
catch (AerospikeException ae)
{
ae.Policy = policy;
ae.Iteration = iteration;
ae.SetInDoubt(IsWrite(), commandSentCounter);
throw;
}
try
{
Connection conn = node.GetConnection(socketTimeout);
try
{
// Set command buffer.
WriteBuffer();
// Send command.
conn.Write(dataBuffer, dataOffset);
commandSentCounter++;
// Parse results.
ParseResult(conn);
// Put connection back in pool.
node.PutConnection(conn);
// Command has completed successfully. Exit method.
return;
}
catch (AerospikeException ae)
{
if (ae.KeepConnection())
{
// Put connection back in pool.
node.PutConnection(conn);
}
else
{
// Close socket to flush out possible garbage. Do not put back in pool.
node.CloseConnection(conn);
}
if (ae.Result == ResultCode.TIMEOUT)
{
// Go through retry logic on server timeout.
exception = new AerospikeException.Timeout(policy, false);
isClientTimeout = false;
}
else
{
throw;
}
}
catch (SocketException se)
{
// Socket errors are considered temporary anomalies.
// Retry after closing connection.
node.CloseConnection(conn);
if (se.SocketErrorCode == SocketError.TimedOut)
{
isClientTimeout = true;
}
else
{
exception = new AerospikeException.Connection(se);
isClientTimeout = false;
}
}
catch (Exception)
{
// All other exceptions are considered fatal. Do not retry.
// Close socket to flush out possible garbage. Do not put back in pool.
node.CloseConnection(conn);
throw;
}
}
catch (SocketException se)
{
// This exception might happen after initial connection succeeded, but
// user login failed with a socket error. Retry.
if (se.SocketErrorCode == SocketError.TimedOut)
{
isClientTimeout = true;
}
else
{
exception = new AerospikeException.Connection(se);
isClientTimeout = false;
}
}
catch (AerospikeException.Connection ce)
{
// Socket connection error has occurred. Retry.
exception = ce;
isClientTimeout = false;
}
catch (AerospikeException ae)
{
ae.Node = node;
ae.Policy = policy;
ae.Iteration = iteration;
ae.SetInDoubt(IsWrite(), commandSentCounter);
throw;
}
// Check maxRetries.
if (iteration > maxRetries)
{
break;
}
if (totalTimeout > 0)
{
// Check for total timeout.
long remaining = (long)deadline.Subtract(DateTime.UtcNow).TotalMilliseconds - policy.sleepBetweenRetries;
if (remaining <= 0)
{
break;
}
if (remaining < totalTimeout)
{
totalTimeout = (int)remaining;
if (socketTimeout > totalTimeout)
{
socketTimeout = totalTimeout;
}
}
}
if (!isClientTimeout && policy.sleepBetweenRetries > 0)
{
// Sleep before trying again.
Util.Sleep(policy.sleepBetweenRetries);
}
iteration++;
if (!PrepareRetry(isClientTimeout || exception.Result != ResultCode.SERVER_NOT_AVAILABLE))
{
// Batch may be retried in separate commands.
if (RetryBatch(cluster, socketTimeout, totalTimeout, deadline, iteration, commandSentCounter))
{
// Batch was retried in separate commands. Complete this command.
return;
}
}
}
// Retries have been exhausted. Throw last exception.
if (isClientTimeout)
{
exception = new AerospikeException.Timeout(policy, true);
}
exception.Node = node;
exception.Policy = policy;
exception.Iteration = iteration;
exception.SetInDoubt(IsWrite(), commandSentCounter);
throw exception;
}
public void Execute(Cluster cluster, Policy policy, Key key, Node node, bool isRead)
{
Partition partition = (key != null)? new Partition(key) : null;
AerospikeException exception = null;
DateTime deadline = DateTime.MinValue;
int socketTimeout = policy.socketTimeout;
int totalTimeout = policy.totalTimeout;
int iteration = 0;
int commandSentCounter = 0;
bool isClientTimeout;
if (totalTimeout > 0)
{
deadline = DateTime.UtcNow.AddMilliseconds(totalTimeout);
if (socketTimeout == 0 || socketTimeout > totalTimeout)
{
socketTimeout = totalTimeout;
}
}
// Execute command until successful, timed out or maximum iterations have been reached.
while (true)
{
try
{
if (partition != null)
{
// Single record command node retrieval.
node = GetNode(cluster, partition, policy.replica, isRead);
}
Connection conn = node.GetConnection(socketTimeout);
try
{
// Set command buffer.
WriteBuffer();
// Check if timeout needs to be changed in send buffer.
if (totalTimeout != policy.totalTimeout)
{
// Reset timeout in send buffer (destined for server) and socket.
ByteUtil.IntToBytes((uint)totalTimeout, dataBuffer, 22);
}
// Send command.
conn.Write(dataBuffer, dataOffset);
commandSentCounter++;
// Parse results.
ParseResult(conn);
// Put connection back in pool.
node.PutConnection(conn);
// Command has completed successfully. Exit method.
return;
}
catch (AerospikeException ae)
{
if (ae.KeepConnection())
{
// Put connection back in pool.
node.PutConnection(conn);
}
else
{
// Close socket to flush out possible garbage. Do not put back in pool.
node.CloseConnection(conn);
}
if (ae.Result == ResultCode.TIMEOUT)
{
// Go through retry logic on server timeout.
exception = new AerospikeException.Timeout(node, policy, iteration + 1, false);
isClientTimeout = false;
if (isRead)
{
base.sequence++;
}
}
else
{
ae.SetInDoubt(isRead, commandSentCounter);
throw;
}
}
catch (SocketException se)
{
// Socket errors are considered temporary anomalies.
// Retry after closing connection.
node.CloseConnection(conn);
if (se.SocketErrorCode == SocketError.TimedOut)
{
isClientTimeout = true;
if (isRead)
{
base.sequence++;
}
}
else
{
exception = new AerospikeException(se);
isClientTimeout = false;
base.sequence++;
}
}
catch (Exception)
{
// All other exceptions are considered fatal. Do not retry.
// Close socket to flush out possible garbage. Do not put back in pool.
node.CloseConnection(conn);
throw;
}
}
catch (AerospikeException.Connection ce)
{
// Socket connection error has occurred. Retry.
exception = ce;
isClientTimeout = false;
base.sequence++;
}
// Check maxRetries.
if (++iteration > policy.maxRetries)
{
break;
}
if (policy.totalTimeout > 0)
{
// Check for total timeout.
long remaining = (long)deadline.Subtract(DateTime.UtcNow).TotalMilliseconds - policy.sleepBetweenRetries;
if (remaining <= 0)
{
break;
}
if (remaining < totalTimeout)
{
totalTimeout = (int)remaining;
if (socketTimeout > totalTimeout)
{
socketTimeout = totalTimeout;
}
}
}
if (!isClientTimeout && policy.sleepBetweenRetries > 0)
{
// Sleep before trying again.
Util.Sleep(policy.sleepBetweenRetries);
}
}
// Retries have been exhausted. Throw last exception.
if (isClientTimeout)
{
exception = new AerospikeException.Timeout(node, policy, iteration, true);
}
exception.SetInDoubt(isRead, commandSentCounter);
throw exception;
}
public void Execute
(
Cluster cluster, Policy policy, Partition partition, Node node, bool isRead,
int socketTimeout, int totalTimeout, DateTime deadline, int iteration, int commandSentCounter
)
{
AerospikeException exception = null;
bool isClientTimeout;
// Execute command until successful, timed out or maximum iterations have been reached.
while (true)
{
if (partition != null)
{
// Single record command node retrieval.
try
{
node = GetNode(cluster, policy, partition, isRead);
}
catch (AerospikeException ae)
{
ae.Iteration = iteration;
ae.SetInDoubt(isRead, commandSentCounter);
throw;
}
}
try
{
Connection conn = node.GetConnection(socketTimeout);
try
{
// Set command buffer.
WriteBuffer();
// Check if timeout needs to be changed in send buffer.
if (totalTimeout != policy.totalTimeout)
{
// Reset timeout in send buffer (destined for server) and socket.
ByteUtil.IntToBytes((uint)totalTimeout, dataBuffer, 22);
}
// Send command.
conn.Write(dataBuffer, dataOffset);
commandSentCounter++;
// Parse results.
ParseResult(conn);
// Put connection back in pool.
node.PutConnection(conn);
// Command has completed successfully. Exit method.
return;
}
catch (AerospikeException ae)
{
if (ae.KeepConnection())
{
// Put connection back in pool.
node.PutConnection(conn);
}
else
{
// Close socket to flush out possible garbage. Do not put back in pool.
node.CloseConnection(conn);
}
if (ae.Result == ResultCode.TIMEOUT)
{
// Go through retry logic on server timeout.
exception = new AerospikeException.Timeout(policy, false);
isClientTimeout = false;
ShiftSequenceOnRead(policy, isRead);
}
else
{
throw;
}
}
catch (SocketException se)
{
// Socket errors are considered temporary anomalies.
// Retry after closing connection.
node.CloseConnection(conn);
if (se.SocketErrorCode == SocketError.TimedOut)
{
isClientTimeout = true;
ShiftSequenceOnRead(policy, isRead);
}
else
{
exception = new AerospikeException(se);
isClientTimeout = false;
base.sequence++;
}
}
catch (Exception)
{
// All other exceptions are considered fatal. Do not retry.
// Close socket to flush out possible garbage. Do not put back in pool.
node.CloseConnection(conn);
throw;
}
}
catch (SocketException se)
{
// This exception might happen after initial connection succeeded, but
// user login failed with a socket error. Retry.
if (se.SocketErrorCode == SocketError.TimedOut)
{
isClientTimeout = true;
ShiftSequenceOnRead(policy, isRead);
}
else
{
exception = new AerospikeException(se);
isClientTimeout = false;
base.sequence++;
}
}
catch (AerospikeException.Connection ce)
{
// Socket connection error has occurred. Retry.
exception = ce;
isClientTimeout = false;
base.sequence++;
}
catch (AerospikeException ae)
{
ae.Node = node;
ae.Iteration = iteration;
ae.SetInDoubt(isRead, commandSentCounter);
throw;
}
// Check maxRetries.
if (iteration > policy.maxRetries)
{
break;
}
if (policy.totalTimeout > 0)
{
// Check for total timeout.
long remaining = (long)deadline.Subtract(DateTime.UtcNow).TotalMilliseconds - policy.sleepBetweenRetries;
if (remaining <= 0)
{
break;
}
if (remaining < totalTimeout)
{
totalTimeout = (int)remaining;
if (socketTimeout > totalTimeout)
{
socketTimeout = totalTimeout;
}
}
}
if (!isClientTimeout && policy.sleepBetweenRetries > 0)
{
// Sleep before trying again.
Util.Sleep(policy.sleepBetweenRetries);
}
iteration++;
if (ShouldRetryBatch() && RetryBatch(cluster, socketTimeout, totalTimeout, deadline, iteration, commandSentCounter))
{
// Batch retried in separate commands. Complete this command.
return;
}
}
// Retries have been exhausted. Throw last exception.
if (isClientTimeout)
{
exception = new AerospikeException.Timeout(policy, true);
}
exception.Node = node;
exception.Iteration = iteration;
exception.SetInDoubt(isRead, commandSentCounter);
throw exception;
}