async Task <MessageData> AddMessageAsync(TaskMessage taskMessage, OrchestrationInstance sourceInstance, SessionBase session)
{
MessageData data;
try
{
// We transfer to a new trace activity ID every time a new outbound queue message is created.
Guid outboundTraceActivityId = Guid.NewGuid();
data = new MessageData(
taskMessage,
outboundTraceActivityId,
this.storageQueue.Name,
session?.GetCurrentEpisode(),
sourceInstance);
data.SequenceNumber = Interlocked.Increment(ref messageSequenceNumber);
string rawContent = await this.messageManager.SerializeMessageDataAsync(data);
CloudQueueMessage queueMessage = new CloudQueueMessage(rawContent);
AnalyticsEventSource.Log.SendingMessage(
outboundTraceActivityId,
this.storageAccountName,
this.settings.TaskHubName,
taskMessage.Event.EventType.ToString(),
Utils.GetTaskEventId(taskMessage.Event),
sourceInstance.InstanceId,
sourceInstance.ExecutionId,
Encoding.Unicode.GetByteCount(rawContent),
data.QueueName /* PartitionId */,
taskMessage.OrchestrationInstance.InstanceId,
taskMessage.OrchestrationInstance.ExecutionId,
data.SequenceNumber,
data.Episode.GetValueOrDefault(-1),
Utils.ExtensionVersion);
await this.storageQueue.AddMessageAsync(
queueMessage,
null /* timeToLive */,
GetVisibilityDelay(taskMessage),
this.QueueRequestOptions,
session?.StorageOperationContext);
this.stats.MessagesSent.Increment();
// Wake up the queue polling thread
this.backoffHelper.Reset();
}
catch (StorageException e)
{
AnalyticsEventSource.Log.MessageFailure(
this.storageAccountName,
this.settings.TaskHubName,
string.Empty /* MessageId */,
sourceInstance.InstanceId,
sourceInstance.ExecutionId,
this.storageQueue.Name,
taskMessage.Event.EventType.ToString(),
Utils.GetTaskEventId(taskMessage.Event),
e.ToString(),
Utils.ExtensionVersion);
throw;
}
finally
{
this.stats.StorageRequests.Increment();
}
return(data);
}
public virtual async Task AbandonMessageAsync(MessageData message, SessionBase session)
{
CloudQueueMessage queueMessage = message.OriginalQueueMessage;
TaskMessage taskMessage = message.TaskMessage;
OrchestrationInstance instance = taskMessage.OrchestrationInstance;
// Exponentially backoff a given queue message until a maximum visibility delay of 10 minutes.
// Once it hits the maximum, log the message as a poison message.
const int maxSecondsToWait = 600;
int numSecondsToWait = Math.Min((int)Math.Pow(2, queueMessage.DequeueCount), maxSecondsToWait);
if (numSecondsToWait == maxSecondsToWait)
{
AnalyticsEventSource.Log.PoisonMessageDetected(
this.storageAccountName,
this.settings.TaskHubName,
queueMessage.Id,
instance.InstanceId,
instance.ExecutionId,
this.storageQueue.Name,
queueMessage.DequeueCount,
Utils.ExtensionVersion);
}
TimeSpan visibilityDelay = TimeSpan.FromSeconds(numSecondsToWait);
AnalyticsEventSource.Log.AbandoningMessage(
this.storageAccountName,
this.settings.TaskHubName,
taskMessage.Event.EventType.ToString(),
Utils.GetTaskEventId(taskMessage.Event),
queueMessage.Id,
instance.InstanceId,
instance.ExecutionId,
this.storageQueue.Name,
message.SequenceNumber,
(int)visibilityDelay.TotalSeconds,
Utils.ExtensionVersion);
try
{
// We "abandon" the message by settings its visibility timeout using an exponential backoff algorithm.
// This allows it to be reprocessed on this node or another node at a later time, hopefully successfully.
await this.storageQueue.UpdateMessageAsync(
queueMessage,
visibilityDelay,
MessageUpdateFields.Visibility,
this.QueueRequestOptions,
session.StorageOperationContext);
this.stats.MessagesUpdated.Increment();
}
catch (Exception e)
{
// Message may have been processed and deleted already.
this.HandleMessagingExceptions(e, message, $"Caller: {nameof(AbandonMessageAsync)}");
}
finally
{
this.stats.StorageRequests.Increment();
}
}
/// <summary>
/// Adds message to a queue
/// </summary>
/// <param name="message">Instance of <see cref="TaskMessage"/></param>
/// <param name="sourceSession">Instance of <see cref="SessionBase"/></param>
/// <returns></returns>
public Task AddMessageAsync(TaskMessage message, SessionBase sourceSession)
{
return(this.AddMessageAsync(message, sourceSession.Instance, sourceSession));
}
public override Task DeleteMessageAsync(MessageData message, SessionBase session)
{
this.stats.PendingOrchestratorMessages.TryRemove(message.OriginalQueueMessage.Id, out _);
return(base.DeleteMessageAsync(message, session));
}
