/// <summary>
/// Given a batch with a set of operations, executes the set and awaits the results of the batch being available.
/// </summary>
/// <param name="context">Context.</param>
/// <param name="batch">The batch to execute.</param>
/// <param name="cancellationToken">A cancellation token for the task</param>
/// <returns>A task that completes when all items in the batch are done.</returns>
public async Task <BoolResult> ExecuteBatchOperationAsync(Context context, IRedisBatch batch, CancellationToken cancellationToken)
{
using (Counters[RedisOperation.All].Start())
using (Counters[RedisOperation.Batch].Start())
using (Counters[batch.Operation].Start())
{
Counters[RedisOperation.BatchSize].Add(batch.BatchSize);
try
{
await _redisRetryStrategy.ExecuteAsync(
async() =>
{
var database = await GetDatabaseAsync(context);
await batch.ExecuteBatchOperationAndGetCompletion(context, database);
},
cancellationToken);
await batch.NotifyConsumersOfSuccess();
RedisOperationSucceeded();
return(BoolResult.Success);
}
catch (Exception ex)
{
batch.NotifyConsumersOfFailure(ex);
HandleRedisExceptionAndResetMultiplexerIfNeeded(context, ex);
return(new BoolResult(ex));
}
}
}
public async Task <BoolResult> ExecuteBatchOperationAsync(Context context, IRedisBatch batch, CancellationToken cancellationToken)
{
var operationContext = new OperationContext(context, cancellationToken);
var result = await operationContext
.CreateOperation(
_tracer,
async() =>
{
using (Counters[RedisOperation.All].Start())
using (Counters[RedisOperation.Batch].Start())
using (Counters[batch.Operation].Start())
{
Counters[RedisOperation.BatchSize].Add(batch.BatchSize);
try
{
await _redisRetryStrategy.ExecuteAsync(
context,
async() =>
{
var database = await GetDatabaseAsync(context);
await batch.ExecuteBatchOperationAndGetCompletion(context, database);
},
cancellationToken,
_configuration.TraceTransientFailures);
await batch.NotifyConsumersOfSuccess();
return(BoolResult.Success);
}
catch (Exception ex)
{
batch.NotifyConsumersOfFailure(ex);
return(new BoolResult(ex));
}
}
})
.TraceErrorsOnlyIfEnabled(
_configuration.TraceOperationFailures,
endMessageFactory: r => $"Database={_configuration.DatabaseName}, ConnectionErrors={_connectionErrorCount}")
.RunAsync();
HandleOperationResult(context, result);
return(result);
}
public async Task <BoolResult> ExecuteBatchOperationAsync(Context context, IRedisBatch batch, CancellationToken cancellationToken)
{
var operationContext = new OperationContext(context, cancellationToken);
using var batchCounter = Counters[RedisOperation.Batch].Start();
using var operationCounter = Counters[batch.Operation].Start();
Counters[RedisOperation.BatchSize].Add(batch.BatchSize);
var result = await PerformRedisOperationAsync(
operationContext,
operation: async(nestedContext, database) =>
{
await batch.ExecuteBatchOperationAndGetCompletion(nestedContext, database, nestedContext.Token);
return(Unit.Void);
},
onSuccess : () => batch.NotifyConsumersOfSuccess(),
onFailure : ex => batch.NotifyConsumersOfFailure(ex),
onCancel : () => batch.NotifyConsumersOfCancellation(),
extraEndMessage : $"Operation={batch.Operation}, ",
operationName : batch.Operation.ToString()
);
return(result.Succeeded ? BoolResult.Success : new BoolResult(result));
}
public async Task <BoolResult> ExecuteBatchOperationAsync(Context context, IRedisBatch batch, CancellationToken cancellationToken)
{
var operationContext = new OperationContext(context, cancellationToken);
var result = await operationContext
.CreateOperation(
_tracer,
async() =>
{
using (Counters[RedisOperation.All].Start())
using (Counters[RedisOperation.Batch].Start())
using (Counters[batch.Operation].Start())
{
Counters[RedisOperation.BatchSize].Add(batch.BatchSize);
try
{
// Need to register the cancellation here and not inside the ExecuteAsync callback,
// because the cancellation can happen before the execution of the given callback.
// And we still need to cancel the batch operations to finish all the tasks associated with them.
using (CancellationTokenRegistration registration = operationContext.Token.Register(
() => { cancelTheBatch(reason: "a given cancellation token is cancelled"); }))
{
await _redisRetryStrategy.ExecuteAsync(
context,
async() =>
{
var(database, databaseClosedCancellationToken) = await GetDatabaseAsync(context);
CancellationTokenSource?linkedCts = null;
if (_configuration.CancelBatchWhenMultiplexerIsClosed)
{
// The database may be closed during a redis call.
// Linking two tokens together and cancelling the batch if one of the cancellations was requested.
// We want to make sure the following: the task returned by this call and the tasks for each and individual
// operation within a batch are cancelled.
// To do that, we need to "Notify" all the batches about the cancellation inside the Register callback
// and ExecuteBatchOperationAndGetCompletion should respect the cancellation token and throw an exception
// if the token is set.
linkedCts = CancellationTokenSource.CreateLinkedTokenSource(databaseClosedCancellationToken);
linkedCts.Token.Register(
() =>
{
string reason = operationContext.Token.IsCancellationRequested
? "a given cancellation token is cancelled"
: "the multiplexer is closed";
cancelTheBatch(reason: reason);
});
// It is fine that the second cancellation token is not passed to retry strategy.
// Retry strategy only retries on redis exceptions and all the rest, like TaskCanceledException or OperationCanceledException
// are be ignored.
}
// We need to dispose the token source to unlink it from the tokens the source was created from.
// This is important, because the database cancellation token can live a long time
// referencing a lot of token sources created here.
using (linkedCts)
{
await batch.ExecuteBatchOperationAndGetCompletion(context, database, linkedCts?.Token ?? CancellationToken.None);
}
},
cancellationToken,
_configuration.TraceTransientFailures);
await batch.NotifyConsumersOfSuccess();
return(BoolResult.Success);
}
}
catch (TaskCanceledException e)
{
return(new BoolResult(e)
{
IsCancelled = true
});
}
catch (OperationCanceledException e)
{
return(new BoolResult(e)
{
IsCancelled = true
});
}
catch (Exception ex)
{
batch.NotifyConsumersOfFailure(ex);
return(new BoolResult(ex));
}
}
})
.TraceErrorsOnlyIfEnabled(
_configuration.TraceOperationFailures,
endMessageFactory: r => $"Operation={batch.Operation}, Database={_configuration.DatabaseName}, ConnectionErrors={_connectionErrorCount}")
.RunAsync();
HandleOperationResult(context, result);
return(result);
void cancelTheBatch(string reason)
{
context.Debug($"Cancelling {batch.Operation} against {batch.DatabaseName} because {reason}.");
batch.NotifyConsumersOfCancellation();
}
}
public async Task <BoolResult> ExecuteBatchOperationAsync(Context context, IRedisBatch batch, CancellationToken cancellationToken)
{
// The cancellation logic in this method is quite complicated.
// We have following "forces" that can cancel the operation:
// 1. A token provided to this method is triggered.
// (if the current operation is no longer needed because we got the result from another redis instance already).
// 2. Operation exceeds a timeout
// 3. A multiplexer is closed and we need to retry with a newly created connection multiplexer.
var operationContext = new OperationContext(context, cancellationToken);
// Cancellation token can be changed in this method so we need another local to avoid re-assigning an argument.
var token = cancellationToken;
var result = await operationContext.PerformOperationWithTimeoutAsync(
_tracer,
async (withTimeoutContext) =>
{
string getCancellationReason(bool multiplexerIsClosed)
{
bool externalTokenIsCancelled = operationContext.Token.IsCancellationRequested;
bool timeoutTokenIsCancelled = withTimeoutContext.Token.IsCancellationRequested;
Contract.Assert(externalTokenIsCancelled || timeoutTokenIsCancelled || multiplexerIsClosed);
// Its possible to have more than one token to be triggered, in this case we'll report based on the check order.
// Have to put '!' at the end of each return statement due to this bug: https://github.com/dotnet/roslyn/issues/42396
// Should be removed once moved to a newer C# compiler version.
if (externalTokenIsCancelled)
{
return("a given cancellation token is cancelled" !);
}
if (timeoutTokenIsCancelled)
{
return($"Operation timed out after {_configuration.OperationTimeout}" !);
}
if (multiplexerIsClosed)
{
return("the multiplexer is closed" !);
}
return("The operation is not cancelled" !);
}
// Now the token is a combination of "external token" and "timeout token"
token = withTimeoutContext.Token;
using (Counters[RedisOperation.All].Start())
using (Counters[RedisOperation.Batch].Start())
using (Counters[batch.Operation].Start())
{
Counters[RedisOperation.BatchSize].Add(batch.BatchSize);
try
{
// Need to register the cancellation here and not inside the ExecuteAsync callback,
// because the cancellation can happen before the execution of the given callback.
// And we still need to cancel the batch operations to finish all the tasks associated with them.
using (token.Register(() => { cancelTheBatch(getCancellationReason(multiplexerIsClosed: false)); }))
{
await _redisRetryStrategy.ExecuteAsync(
withTimeoutContext,
async() =>
{
var(database, databaseClosedCancellationToken) = await GetDatabaseAsync(withTimeoutContext);
CancellationTokenSource?linkedCts = null;
if (_configuration.CancelBatchWhenMultiplexerIsClosed)
{
// The database may be closed during a redis call.
// Linking two tokens together and cancelling the batch if one of the cancellations was requested.
// We want to make sure the following: the task returned by this call and the tasks for each and individual
// operation within a batch are cancelled.
// To do that, we need to "Notify" all the batches about the cancellation inside the Register callback
// and ExecuteBatchOperationAndGetCompletion should respect the cancellation token and throw an exception
// if the token is set.
linkedCts = CancellationTokenSource.CreateLinkedTokenSource(databaseClosedCancellationToken, withTimeoutContext.Token);
linkedCts.Token.Register(
() =>
{
cancelTheBatch(getCancellationReason(multiplexerIsClosed: databaseClosedCancellationToken.IsCancellationRequested));
});
// Now the token is a combination of "external token", "timeout token" and "database is closed token"
token = linkedCts.Token;
// It is fine that the second cancellation token is not passed to retry strategy.
// Retry strategy only retries on redis exceptions and all the rest, like TaskCanceledException or OperationCanceledException
// are be ignored.
}
// We need to dispose the token source to unlink it from the tokens the source was created from.
// This is important, because the database cancellation token can live a long time
// referencing a lot of token sources created here.
using (linkedCts)
{
await batch.ExecuteBatchOperationAndGetCompletion(withTimeoutContext, database, token);
}
},
token);
await batch.NotifyConsumersOfSuccess();
return(BoolResult.Success);
}
}
catch (TaskCanceledException e)
{
// Don't have to cancel batch here, because we track the cancellation already and call 'cancelBatch' if needed
return(new BoolResult(e)
{
IsCancelled = true
});
}
catch (OperationCanceledException e)
{
// The same applies to OperationCanceledException as for TaskCanceledException
return(new BoolResult(e)
{
IsCancelled = true
});
}
catch (Exception ex)
{
batch.NotifyConsumersOfFailure(ex);
return(new BoolResult(ex)
{
IsCancelled = token.IsCancellationRequested
});
}
}
},
// Tracing errors all the time. They're not happening too frequently and its useful to know about all of them.
traceErrorsOnly: true,
traceOperationStarted : false,
//traceOperationFinished: _configuration.TraceOperationFailures,
extraEndMessage : r => $"Operation={batch.Operation}, Database={_configuration.DatabaseName}, ConnectionErrors={_connectionErrorCount}, IsCancelled={token.IsCancellationRequested}",
timeout : _configuration.OperationTimeout);
HandleOperationResult(context, result);
return(result);
void cancelTheBatch(string reason)
{
context.Debug($"Cancelling {batch.Operation} against {batch.DatabaseName} because {reason}.");
batch.NotifyConsumersOfCancellation();
}
}