async Task DeleteAndDispatch(DelayedMessageEntity delayedMessage, CancellationToken cancellationToken)
{
try
{
var delete = TableOperation.Delete(delayedMessage);
if (isAtMostOnce)
{
// delete first, then dispatch
await delayedDeliveryTable.ExecuteAsync(delete, cancellationToken).ConfigureAwait(false);
await SafeDispatch(delayedMessage, cancellationToken).ConfigureAwait(false);
}
else
{
// dispatch first, then delete
await SafeDispatch(delayedMessage, cancellationToken).ConfigureAwait(false);
await delayedDeliveryTable.ExecuteAsync(delete, cancellationToken).ConfigureAwait(false);
}
}
catch (Exception ex) when(!ex.IsCausedBy(cancellationToken))
{
// just log and move on with the rest
Logger.Warn(
$"Failed at dispatching the delayed message with PartitionKey:'{delayedMessage.PartitionKey}' RowKey: '{delayedMessage.RowKey}' message ID: '{delayedMessage.MessageId}'",
ex);
}
}
Task ScheduleAt(UnicastTransportOperation operation, DateTimeOffset date, CancellationToken cancellationToken)
{
var delayedMessageEntity = new DelayedMessageEntity
{
PartitionKey = DelayedMessageEntity.GetPartitionKey(date),
RowKey = $"{DelayedMessageEntity.GetRawRowKeyPrefix(date)}_{Guid.NewGuid():N}",
};
delayedMessageEntity.SetOperation(operation);
return(delayedMessagesTable.ExecuteAsync(TableOperation.Insert(delayedMessageEntity), null, null, cancellationToken));
}
async Task SpinOnce(CancellationToken cancellationToken)
{
var now = DateTimeOffset.UtcNow;
Logger.Debug($"Polling for delayed messages at {now}.");
var query = new TableQuery <DelayedMessageEntity>
{
FilterString = $"(PartitionKey le '{DelayedMessageEntity.GetPartitionKey(now)}') and (RowKey le '{DelayedMessageEntity.GetRawRowKeyPrefix(now)}')",
TakeCount = DelayedMessagesProcessedAtOnce // max batch size
};
var delayedMessages = await delayedDeliveryTable.QueryUpTo(query, DelayedMessagesProcessedAtOnce, cancellationToken)
.ConfigureAwait(false);
if (await lockManager.TryLockOrRenew(cancellationToken).ConfigureAwait(false) == false)
{
return;
}
Stopwatch stopwatch = null;
var delayedMessagesCount = delayedMessages.Count;
var dispatchOperations = new List <Task>(delayedMessagesCount);
foreach (var delayedMessage in delayedMessages)
{
cancellationToken.ThrowIfCancellationRequested();
// only allocate if needed
stopwatch = stopwatch ?? Stopwatch.StartNew();
// after half check if the lease is active
if (stopwatch.Elapsed > HalfOfLeaseLength)
{
if (await lockManager.TryLockOrRenew(cancellationToken).ConfigureAwait(false) == false)
{
return;
}
stopwatch.Reset();
}
// deliberately not using the dispatchers batching capability because every delayed message dispatch should be independent
dispatchOperations.Add(DeleteAndDispatch(delayedMessage, cancellationToken));
}
if (delayedMessagesCount > 0)
{
await Task.WhenAll(dispatchOperations).ConfigureAwait(false);
}
await backoffStrategy.OnBatch(delayedMessagesCount, cancellationToken).ConfigureAwait(false);
}
async Task SafeDispatch(DelayedMessageEntity delayedMessage, CancellationToken cancellationToken)
{
var operation = delayedMessage.GetOperation();
try
{
await dispatcher.Send(operation, cancellationToken).ConfigureAwait(false);
}
catch (Exception)
{
// if send fails for any reason
await dispatcher.Send(CreateOperationForErrorQueue(operation), cancellationToken).ConfigureAwait(false);
}
}
public Task ScheduleAt(UnicastTransportOperation operation, DateTimeOffset date, CancellationToken cancellationToken = default)
{
if (!enabled)
{
throw new Exception("Native delayed deliveries are not enabled.");
}
var delayedMessageEntity = new DelayedMessageEntity
{
PartitionKey = DelayedMessageEntity.GetPartitionKey(date),
RowKey = $"{DelayedMessageEntity.GetRawRowKeyPrefix(date)}_{Guid.NewGuid():N}",
};
delayedMessageEntity.SetOperation(operation);
return(delayedMessageStorageTable.ExecuteAsync(TableOperation.Insert(delayedMessageEntity), null, null, cancellationToken));
}
async Task SafeDispatch(DelayedMessageEntity delayedMessage, CancellationToken cancellationToken)
{
if (Logger.IsDebugEnabled)
{
Logger.DebugFormat("Dispatching delayed message ID: '{0}'", delayedMessage.MessageId);
}
var operation = delayedMessage.GetOperation();
try
{
await dispatcher.Send(operation, cancellationToken).ConfigureAwait(false);
}
catch (Exception ex) when(!ex.IsCausedBy(cancellationToken))
{
// if send fails for any reason
Logger.Warn($"Failed to send the delayed message with PartitionKey:'{delayedMessage.PartitionKey}' RowKey: '{delayedMessage.RowKey}' message ID: '{delayedMessage.MessageId}'", ex);
await dispatcher.Send(CreateOperationForErrorQueue(operation), cancellationToken).ConfigureAwait(false);
}
}
async Task SpinOnce(CancellationToken cancellationToken)
{
var now = NativeDelayDelivery.UtcNow;
if (cancellationToken.IsCancellationRequested)
{
return;
}
Logger.DebugFormat("Polling for delayed messages at {0}.", now);
var query = new TableQuery <DelayedMessageEntity>
{
FilterString = $"(PartitionKey le '{DelayedMessageEntity.GetPartitionKey(now)}') and (RowKey le '{DelayedMessageEntity.GetRawRowKeyPrefix(now)}')",
TakeCount = DelayedMessagesProcessedAtOnce // max batch size
};
var delayedMessages = await delayedDeliveryTable.ExecuteQueryAsync(query, DelayedMessagesProcessedAtOnce, cancellationToken)
.ConfigureAwait(false);
if (await TryLease(cancellationToken).ConfigureAwait(false) == false)
{
return;
}
var stopwatch = Stopwatch.StartNew();
foreach (var delayedMessage in delayedMessages)
{
if (cancellationToken.IsCancellationRequested)
{
return;
}
// after half check if the lease is active
if (stopwatch.Elapsed > HalfOfLeaseLength)
{
if (await TryLease(cancellationToken).ConfigureAwait(false) == false)
{
return;
}
stopwatch.Reset();
}
try
{
var delete = TableOperation.Delete(delayedMessage);
if (isAtMostOnce)
{
// delete first, then dispatch
// TODO: raise an issue with the client library about missing overload accepting operation and cancellation token only
await delayedDeliveryTable.ExecuteAsync(delete, null, null, cancellationToken).ConfigureAwait(false);
await SafeDispatch(delayedMessage, cancellationToken).ConfigureAwait(false);
}
else
{
// dispatch first, then delete
await SafeDispatch(delayedMessage, cancellationToken).ConfigureAwait(false);
// TODO: raise an issue with the client library about missing overload accepting operation and cancellation token only
await delayedDeliveryTable.ExecuteAsync(delete, null, null, cancellationToken).ConfigureAwait(false);
}
}
catch (Exception exception)
{
// just log and move on with the rest
Logger.Warn($"Failed at dispatching the delayed message with PartitionKey:'{delayedMessage.PartitionKey}' RowKey: '{delayedMessage.RowKey}' message ID: '{delayedMessage.MessageId}'", exception);
}
}
await backoffStrategy.OnBatch(delayedMessages.Count, cancellationToken).ConfigureAwait(false);
}
