public async Task AbandonMessageLogsEvents()
{
var mockLogger = new Mock <ServiceBusEventSource>();
var mockTransportReceiver = new Mock <TransportReceiver>();
var mockConnection = GetMockConnection(mockTransportReceiver);
var receiver = new ServiceBusReceiver(
mockConnection.Object,
"queueName",
false,
new ServiceBusPlugin[] { },
new ServiceBusReceiverOptions())
{
Logger = mockLogger.Object
};
var msg = new ServiceBusReceivedMessage()
{
LockTokenGuid = Guid.NewGuid()
};
await receiver.AbandonMessageAsync(msg);
mockLogger
.Verify(
log => log.AbandonMessageStart(
receiver.Identifier,
1,
msg.LockToken),
Times.Once);
mockLogger
.Verify(
log => log.AbandonMessageComplete(
receiver.Identifier),
Times.Once);
}
public async Task AbandonMessage()
{
await using (var scope = await ServiceBusScope.CreateWithQueue(enablePartitioning: false, enableSession: false))
{
string connectionString = TestEnvironment.ServiceBusConnectionString;
string queueName = scope.QueueName;
// since ServiceBusClient implements IAsyncDisposable we create it with "await using"
await using var client = new ServiceBusClient(connectionString);
// create the sender
ServiceBusSender sender = client.CreateSender(queueName);
// create a message that we can send
ServiceBusMessage message = new ServiceBusMessage(Encoding.UTF8.GetBytes("Hello world!"));
// send the message
await sender.SendMessageAsync(message);
// create a receiver that we can use to receive and settle the message
ServiceBusReceiver receiver = client.CreateReceiver(queueName);
#region Snippet:ServiceBusAbandonMessage
ServiceBusReceivedMessage receivedMessage = await receiver.ReceiveMessageAsync();
// abandon the message, thereby releasing the lock and allowing it to be received again by this or other receivers
await receiver.AbandonMessageAsync(receivedMessage);
#endregion
Assert.IsNotNull(GetNoRetryClient().CreateReceiver(queueName).ReceiveMessageAsync());
}
}
public async Task AbandonMessages(bool useSpecificSession)
{
await using (var scope = await ServiceBusScope.CreateWithQueue(enablePartitioning: false, enableSession: true))
{
await using var client = new ServiceBusClient(TestEnvironment.ServiceBusConnectionString);
ServiceBusSender sender = client.CreateSender(scope.QueueName);
var messageCount = 10;
var sessionId = "sessionId1";
using ServiceBusMessageBatch batch = await sender.CreateMessageBatchAsync();
IEnumerable <ServiceBusMessage> messages = AddMessages(batch, messageCount, sessionId).AsEnumerable <ServiceBusMessage>();
await sender.SendMessagesAsync(batch);
ServiceBusReceiver receiver = await client.CreateSessionReceiverAsync(
scope.QueueName,
new ServiceBusSessionReceiverOptions
{
SessionId = useSpecificSession ? sessionId : null
});
var messageEnum = messages.GetEnumerator();
var remainingMessages = messageCount;
IList <ServiceBusReceivedMessage> receivedMessages = new List <ServiceBusReceivedMessage>();
while (remainingMessages > 0)
{
foreach (var msg in await receiver.ReceiveMessagesAsync(remainingMessages))
{
remainingMessages--;
messageEnum.MoveNext();
Assert.AreEqual(messageEnum.Current.MessageId, msg.MessageId);
Assert.AreEqual(messageEnum.Current.SessionId, msg.SessionId);
receivedMessages.Add(msg);
Assert.AreEqual(msg.DeliveryCount, 1);
}
}
Assert.AreEqual(0, remainingMessages);
// don't abandon in the receive loop
// as this would make the message available to be immediately received again
foreach (var msg in receivedMessages)
{
await receiver.AbandonMessageAsync(msg);
}
messageEnum.Reset();
var receivedMessageCount = 0;
foreach (var item in await receiver.PeekMessagesAsync(messageCount))
{
receivedMessageCount++;
messageEnum.MoveNext();
Assert.AreEqual(messageEnum.Current.MessageId, item.MessageId);
Assert.AreEqual(messageEnum.Current.SessionId, item.SessionId);
}
Assert.AreEqual(messageCount, receivedMessageCount);
}
}
private static async Task ExceedMaxDeliveryAsync(string queueName)
{
ServiceBusReceiver receiver = _client.CreateReceiver(queueName);
while (true)
{
// Ask the broker to return any message readily available or return with no
// result after 2 seconds (allowing for clients with great network latency)
ServiceBusReceivedMessage msg = await receiver.ReceiveMessageAsync(TimeSpan.FromSeconds(2));
if (msg != null)
{
// Now we immediately abandon the message, which increments the DeliveryCount
Console.WriteLine($"Picked up message: Id = {msg.MessageId}; DeliveryCount {msg.DeliveryCount}");
await receiver.AbandonMessageAsync(msg);
}
else
{
// Once the system moves the message to the DLQ, the main queue is empty
// and the loop exits as ReceiveAsync returns null.
break;
}
}
// For picking up the message from a DLQ, we make a receiver just like for a
// regular queue. We could also use QueueClient and a registered handler here.
// The required path is constructed with the EntityNameHelper.FormatDeadLetterPath()
// helper method, and always follows the pattern "{entity}/$DeadLetterQueue",
// meaning that for a queue "Q1", the path is "Q1/$DeadLetterQueue" and for a
// topic "T1" and subscription "S1", the path is "T1/Subscriptions/S1/$DeadLetterQueue"
ServiceBusReceiver deadletterReceiver = _client.CreateReceiver(queueName,
new ServiceBusReceiverOptions {
SubQueue = SubQueue.DeadLetter
});
while (true)
{
// receive a message
ServiceBusReceivedMessage message =
await deadletterReceiver.ReceiveMessageAsync(TimeSpan.FromSeconds(10));
if (message != null)
{
// write out the message deadletter information
Console.WriteLine("Deadletter message:");
Console.WriteLine($"DeadLetterReason = {message.DeadLetterReason}");
Console.WriteLine($"DeadLetterErrorDescription = {message.DeadLetterErrorDescription}");
// complete and therefore remove the message from the DLQ
await deadletterReceiver.CompleteMessageAsync(message);
}
else
{
// DLQ was empty on last receive attempt
break;
}
}
}
public static async Task SafeAbandonMessageAsync(this ServiceBusReceiver messageReceiver, ServiceBusReceivedMessage message, TransportTransactionMode transportTransactionMode, Transaction committableTransaction = null, CancellationToken cancellationToken = default)
{
if (transportTransactionMode != TransportTransactionMode.None)
{
using (var scope = committableTransaction.ToScope())
{
await messageReceiver.AbandonMessageAsync(message, cancellationToken : cancellationToken).ConfigureAwait(false);
scope.Complete();
}
}
}
private Task RecieveMessages(ServiceBusReceivedMessage sbMessage, CancellationToken token)
{
if (sbMessage.DeliveryCount >= maxDequeueCount)
{
return(reciever.DeadLetterMessageAsync(sbMessage, sbMessage.LockToken));
}
IActorMessage msg = null;
if (typeof(MsgType) == typeof(byte[]))
{
msg = sbMessage.Body.ToArray().ToActorMessage();
}
else if (typeof(MsgType) == typeof(string))
{
msg = Encoding.UTF8.GetString(sbMessage.Body.ToArray()).ToActorMessage();
}
else
{
byte[] msgBytes = sbMessage.Body.ToArray();
var t = Telegraph.Instance.Ask(new DeserializeMessage <MsgType>(msgBytes));
msg = t.Result as IActorMessage;
}
msgQueue.Enqueue(msg);
if (null == msg.Status)
{
msg.Status = new TaskCompletionSource <IActorMessage>();
}
// Complete the message so that it is not received again.
// This can be done only if the queueClient is created in ReceiveMode.PeekLock mode (which is default).
msg.Status.Task.ContinueWith(p =>
{
// Note: Use the cancellationToken passed as necessary to determine if the queueClient has already been closed.
// If queueClient has already been Closed, you may chose to not call CompleteAsync() or AbandonAsync() etc. calls
// to avoid unnecessary exceptions.
if (token.IsCancellationRequested)
{
reciever.AbandonMessageAsync(sbMessage, null, token).Wait();
}
else
{
reciever.CompleteMessageAsync(sbMessage, token).Wait();
}
});
return(msg.Status.Task);
}
public void AbandonMessageExceptionLogsEvents()
{
var mockLogger = new Mock <ServiceBusEventSource>();
var mockTransportReceiver = new Mock <TransportReceiver>();
var mockConnection = GetMockConnection(mockTransportReceiver);
mockTransportReceiver.Setup(
transportReceiver => transportReceiver.AbandonAsync(
It.IsAny <string>(),
It.IsAny <IDictionary <string, object> >(),
It.IsAny <CancellationToken>()))
.Throws(new Exception());
var receiver = new ServiceBusReceiver(
mockConnection.Object,
"queueName",
false,
new ServiceBusPlugin[] { },
new ServiceBusReceiverOptions())
{
Logger = mockLogger.Object
};
var msg = new ServiceBusReceivedMessage()
{
LockTokenGuid = Guid.NewGuid()
};
Assert.That(
async() => await receiver.AbandonMessageAsync(msg),
Throws.InstanceOf <Exception>());
mockLogger
.Verify(
log => log.AbandonMessageStart(
receiver.Identifier,
1,
msg.LockToken),
Times.Once);
mockLogger
.Verify(
log => log.AbandonMessageException(
receiver.Identifier,
It.IsAny <string>()),
Times.Once);
}
///<inheritdoc cref="ServiceBusReceiver.AbandonMessageAsync(ServiceBusReceivedMessage, IDictionary{string, object}, CancellationToken)"/>
public virtual async Task AbandonMessageAsync(
ServiceBusReceivedMessage message,
IDictionary <string, object> propertiesToModify = default,
CancellationToken cancellationToken = default)
{
if (_receiver != null)
{
await _receiver.AbandonMessageAsync(message, propertiesToModify, cancellationToken).ConfigureAwait(false);
}
else if (_eventArgs != null)
{
await _eventArgs.AbandonMessageAsync(message, propertiesToModify, cancellationToken).ConfigureAwait(false);
}
else
{
await _sessionEventArgs.AbandonMessageAsync(message, propertiesToModify, cancellationToken).ConfigureAwait(false);
}
}
public static async Task CompleteOrAbandonMessageAsync(string connectionString, string queueName, bool abandon)
{
await using var client = new ServiceBusClient(connectionString);
// create a receiver that we can use to receive the message
ServiceBusReceiver receiver = client.CreateReceiver(queueName);
// the received message is a different type as it contains some service set properties
ServiceBusReceivedMessage receivedMessage = await receiver.ReceiveMessageAsync();
Console.WriteLine($"Message received {receivedMessage.Body} to be abandoned {abandon}");
if (!abandon) // complete the message, thereby deleting it from the service
{
await receiver.CompleteMessageAsync(receivedMessage);
}
else // abandon the message, thereby releasing the lock and allowing it to be received again by this or other receivers
{
await receiver.AbandonMessageAsync(receivedMessage);
}
}
private async Task RunBatchReceiveLoopAsync(CancellationToken cancellationToken)
{
ServiceBusClient sessionClient = null;
ServiceBusReceiver receiver = null;
if (_isSessionsEnabled)
{
sessionClient = _client.Value;
}
else
{
receiver = _batchReceiver.Value;
}
// The batch receive loop below only executes functions at a concurrency level of 1,
// so we don't need to do anything special when DynamicConcurrency is enabled. If in
// the future we make this loop concurrent, we'll have to check with ConcurrencyManager.
while (true)
{
try
{
if (cancellationToken.IsCancellationRequested)
{
_logger.LogInformation($"Message processing has been stopped or cancelled ({_details.Value})");
return;
}
if (_isSessionsEnabled && (receiver == null || receiver.IsClosed))
{
try
{
receiver = await sessionClient.AcceptNextSessionAsync(
_entityPath,
new ServiceBusSessionReceiverOptions
{
PrefetchCount = _serviceBusOptions.PrefetchCount
},
cancellationToken).ConfigureAwait(false);
}
catch (ServiceBusException ex)
when(ex.Reason == ServiceBusFailureReason.ServiceTimeout)
{
// it's expected if the entity is empty, try next time
continue;
}
}
IReadOnlyList <ServiceBusReceivedMessage> messages =
await receiver.ReceiveMessagesAsync(
_serviceBusOptions.MaxMessageBatchSize,
cancellationToken : cancellationToken).AwaitWithCancellation(cancellationToken);
if (messages.Count > 0)
{
var messageActions = _isSessionsEnabled
? new ServiceBusSessionMessageActions((ServiceBusSessionReceiver)receiver)
: new ServiceBusMessageActions(receiver);
var receiveActions = new ServiceBusReceiveActions(receiver);
ServiceBusReceivedMessage[] messagesArray = messages.ToArray();
ServiceBusTriggerInput input = ServiceBusTriggerInput.CreateBatch(
messagesArray,
messageActions,
receiveActions,
_client.Value);
FunctionResult result = await _triggerExecutor.TryExecuteAsync(input.GetTriggerFunctionData(), cancellationToken).ConfigureAwait(false);
receiveActions.EndExecutionScope();
var processedMessages = messagesArray.Concat(receiveActions.Messages.Keys);
// Complete batch of messages only if the execution was successful
if (_autoCompleteMessages && result.Succeeded)
{
List <Task> completeTasks = new List <Task>();
foreach (ServiceBusReceivedMessage message in processedMessages)
{
// skip messages that were settled in the user's function
if (input.MessageActions.SettledMessages.ContainsKey(message))
{
continue;
}
// Pass CancellationToken.None to allow autocompletion to finish even when shutting down
completeTasks.Add(receiver.CompleteMessageAsync(message, CancellationToken.None));
}
await Task.WhenAll(completeTasks).ConfigureAwait(false);
}
else if (!result.Succeeded)
{
// For failed executions, we abandon the messages regardless of the autoCompleteMessages configuration.
// This matches the behavior that happens for single dispatch functions as the processor does the same thing
// in the Service Bus SDK.
List <Task> abandonTasks = new List <Task>();
foreach (ServiceBusReceivedMessage message in processedMessages)
{
// skip messages that were settled in the user's function
if (input.MessageActions.SettledMessages.ContainsKey(message))
{
continue;
}
// Pass CancellationToken.None to allow abandon to finish even when shutting down
abandonTasks.Add(receiver.AbandonMessageAsync(message, cancellationToken: CancellationToken.None));
}
await Task.WhenAll(abandonTasks).ConfigureAwait(false);
}
if (_isSessionsEnabled)
{
if (((ServiceBusSessionMessageActions)messageActions).ShouldReleaseSession)
{
// Use CancellationToken.None to attempt to close the receiver even when shutting down
await receiver.CloseAsync(CancellationToken.None).ConfigureAwait(false);
}
}
}
else
{
// Close the session and release the session lock after draining all messages for the accepted session.
if (_isSessionsEnabled)
{
// Use CancellationToken.None to attempt to close the receiver even when shutting down
await receiver.CloseAsync(CancellationToken.None).ConfigureAwait(false);
}
}
}
catch (ObjectDisposedException)
{
// Ignore as we are stopping the host
}
catch (OperationCanceledException)
when(cancellationToken.IsCancellationRequested)
{
// Ignore as we are stopping the host
_logger.LogInformation($"Message processing has been stopped or cancelled ({_details.Value})");
}
catch (Exception ex)
{
// Log another exception
_logger.LogError(ex, $"An unhandled exception occurred in the message batch receive loop ({_details.Value})");
if (_isSessionsEnabled && receiver != null)
{
// Attempt to close the session and release session lock to accept a new session on the next loop iteration
try
{
// Use CancellationToken.None to attempt to close the receiver even when shutting down
await receiver.CloseAsync(CancellationToken.None).ConfigureAwait(false);
}
catch
{
// Best effort
receiver = null;
}
}
}
}
}
public async Task Run([TimerTrigger("*/10 * * * * *")] TimerInfo myTimer, ILogger log)
{
Stopwatch runStopwatch = new Stopwatch();
runStopwatch.Start();
string runId = Guid.NewGuid().ToString();
log.LogInformation($"TimersSubmissionConsumer function executing: {runId}");
ServiceBusReceiver serviceBusReceiver = _submissionServiceBusService.SubmissionServiceBusClient.CreateReceiver("submission");
var messages = await _submissionServiceBusService.ReceiveMessagesAsync(serviceBusReceiver);
log.LogInformation($"Received message count: {messages.Count} : {runId}");
foreach (var message in messages)
{
Stopwatch stopStopwatch = new Stopwatch();
stopStopwatch.Start();
await serviceBusReceiver.RenewMessageLockAsync(message);
SubmissionMessage submissionMessage = DeserializeQueueItem(log, Encoding.UTF8.GetString(message.Body));
if (submissionMessage == null)
{
await serviceBusReceiver.DeadLetterMessageAsync(message);
continue;
}
// Get Stop
Stop stop = await GetStop(log, submissionMessage.StopId, runId);
if (stop == null)
{
log.LogWarning($"Failed to find stop: {submissionMessage.StopId} : {runId}");
await serviceBusReceiver.AbandonMessageAsync(message); // allows for retry to occur.
continue;
}
DateTime dateSubmitted = DateTime.UtcNow;
// Get File Name
string fileName = GetFileName(log, submissionMessage.SubmissionId, dateSubmitted, stop.Ori, stop.Id, runId);
log.LogInformation($"Using filename: {fileName} : {runId}");
if (fileName == null)
{
await serviceBusReceiver.AbandonMessageAsync(message); // allows for retry to occur.
continue;
}
// Get Doj Stop
DojStop dojStop = GetDojStop(log, stop, runId);
if (dojStop == null)
{
// if the cast error fails report, retry the message
if (!await HandledDojCastError(log, stop, dateSubmitted, fileName, submissionMessage.SubmissionId, runId))
{
await serviceBusReceiver.AbandonMessageAsync(message); // allows for retry to occur.
continue;
}
else
{
await serviceBusReceiver.CompleteMessageAsync(message); // message complete
continue;
}
}
//Get File Bytes
byte[] bytes = GetFileBytes(log, dojStop, runId);
if (bytes == null)
{
log.LogWarning($"Failed to get file contents: {dojStop.LEARecordID} : {runId}");
await serviceBusReceiver.AbandonMessageAsync(message); // allows for retry to occur.
continue;
}
//Upload Blob
if (!await UploadBlob(log, bytes, fileName, stop.Id, runId))
{
log.LogWarning($"Failed to upload blob: {stop.Id} : {runId}");
await serviceBusReceiver.AbandonMessageAsync(message); // allows for retry to occur.
continue;
}
if (!await UploadSftpFile(log, bytes, fileName, stop.Id, runId, stop))
{
log.LogWarning($"Failed to upload to FTP: {stop.Id} : {runId}");
await RemoveBlob(log, fileName, stop.Id, runId); // delete the blob to clean up the failed run
await serviceBusReceiver.AbandonMessageAsync(message); // allows for retry to occur.
continue;
}
if (!await HandleDojSubmitSuccess(log, stop, dateSubmitted, submissionMessage.SubmissionId, fileName, runId))
{
log.LogWarning($"Failed to handle doj submit success: {stop.Id} : {runId}");
RemoveSftpFile(log, fileName, stop.Id, runId); // remove the file from the SFTP server so it doesnt get duplicated.
await serviceBusReceiver.AbandonMessageAsync(message); // allows for retry to occur.
continue;
}
await serviceBusReceiver.CompleteMessageAsync(message); // message complete
stopStopwatch.Stop();
log.LogInformation($"Finished processing STOP : {stop.Id} : {stopStopwatch.ElapsedMilliseconds} : {runId}");
}
runStopwatch.Stop();
log.LogInformation($"TimersSubmissionConsumer finished: {runStopwatch.ElapsedMilliseconds} : {runId}");
}
internal void StartMessageBatchReceiver(CancellationToken cancellationToken)
{
ServiceBusClient sessionClient = null;
ServiceBusReceiver receiver = null;
if (_isSessionsEnabled)
{
sessionClient = _sessionClient.Value;
}
else
{
receiver = BatchReceiver;
}
Task.Run(async() =>
{
while (true)
{
try
{
if (!_started || cancellationToken.IsCancellationRequested)
{
_logger.LogInformation("Message processing has been stopped or cancelled");
return;
}
if (_isSessionsEnabled && (receiver == null || receiver.IsClosed))
{
try
{
receiver = await sessionClient.AcceptNextSessionAsync(_entityPath, new ServiceBusSessionReceiverOptions
{
PrefetchCount = _serviceBusOptions.PrefetchCount
}).ConfigureAwait(false);
}
catch (ServiceBusException ex)
when(ex.Reason == ServiceBusFailureReason.ServiceTimeout)
{
// it's expected if the entity is empty, try next time
continue;
}
}
IReadOnlyList <ServiceBusReceivedMessage> messages =
await receiver.ReceiveMessagesAsync(_serviceBusOptions.MaxMessages).ConfigureAwait(false);
if (messages != null)
{
ServiceBusReceivedMessage[] messagesArray = messages.ToArray();
ServiceBusTriggerInput input = ServiceBusTriggerInput.CreateBatch(messagesArray);
if (_isSessionsEnabled)
{
input.MessageActions = new ServiceBusSessionMessageActions((ServiceBusSessionReceiver)receiver);
}
else
{
input.MessageActions = new ServiceBusMessageActions(receiver);
}
FunctionResult result = await _triggerExecutor.TryExecuteAsync(input.GetTriggerFunctionData(), cancellationToken).ConfigureAwait(false);
if (cancellationToken.IsCancellationRequested)
{
return;
}
// Complete batch of messages only if the execution was successful
if (_serviceBusOptions.AutoCompleteMessages && _started)
{
if (result.Succeeded)
{
List <Task> completeTasks = new List <Task>();
foreach (ServiceBusReceivedMessage message in messagesArray)
{
completeTasks.Add(receiver.CompleteMessageAsync(message));
}
await Task.WhenAll(completeTasks).ConfigureAwait(false);
}
else
{
List <Task> abandonTasks = new List <Task>();
foreach (ServiceBusReceivedMessage message in messagesArray)
{
abandonTasks.Add(receiver.AbandonMessageAsync(message));
}
await Task.WhenAll(abandonTasks).ConfigureAwait(false);
}
}
}
else
{
// Close the session and release the session lock after draining all messages for the accepted session.
if (_isSessionsEnabled)
{
await receiver.CloseAsync().ConfigureAwait(false);
}
}
}
catch (ObjectDisposedException)
{
// Ignore as we are stopping the host
}
catch (Exception ex)
{
// Log another exception
_logger.LogError(ex, $"An unhandled exception occurred in the message batch receive loop");
if (_isSessionsEnabled && receiver != null)
{
// Attempt to close the session and release session lock to accept a new session on the next loop iteration
try
{
await receiver.CloseAsync().ConfigureAwait(false);
}
catch
{
// Best effort
receiver = null;
}
}
}
}
}, cancellationToken);
}
public async Task SenderReceiverActivities(bool useSessions)
{
await using (var scope = await ServiceBusScope.CreateWithQueue(enablePartitioning: false, enableSession: useSessions))
{
var client = new ServiceBusClient(TestEnvironment.ServiceBusConnectionString);
ServiceBusSender sender = client.CreateSender(scope.QueueName);
string sessionId = null;
if (useSessions)
{
sessionId = "sessionId";
}
int numMessages = 5;
var msgs = ServiceBusTestUtilities.GetMessages(numMessages, sessionId);
await sender.SendMessagesAsync(msgs);
Activity[] sendActivities = AssertSendActivities(useSessions, sender, msgs);
ServiceBusReceiver receiver = null;
if (useSessions)
{
receiver = await client.AcceptNextSessionAsync(scope.QueueName);
}
else
{
receiver = client.CreateReceiver(scope.QueueName);
}
var remaining = numMessages;
List <ServiceBusReceivedMessage> receivedMsgs = new List <ServiceBusReceivedMessage>();
while (remaining > 0)
{
// loop in case we don't receive all messages in one attempt
var received = await receiver.ReceiveMessagesAsync(remaining);
receivedMsgs.AddRange(received);
var receiveScope = _listener.AssertAndRemoveScope(DiagnosticProperty.ReceiveActivityName);
AssertCommonTags(receiveScope.Activity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
var receiveLinkedActivities = receiveScope.LinkedActivities;
for (int i = 0; i < receiveLinkedActivities.Count; i++)
{
Assert.AreEqual(sendActivities[i].ParentId, receiveLinkedActivities[i].ParentId);
}
remaining -= received.Count;
}
var msgIndex = 0;
var completed = receivedMsgs[msgIndex];
await receiver.CompleteMessageAsync(completed);
var completeScope = _listener.AssertAndRemoveScope(DiagnosticProperty.CompleteActivityName);
AssertCommonTags(completeScope.Activity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
var deferred = receivedMsgs[++msgIndex];
await receiver.DeferMessageAsync(deferred);
var deferredScope = _listener.AssertAndRemoveScope(DiagnosticProperty.DeferActivityName);
AssertCommonTags(deferredScope.Activity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
var deadLettered = receivedMsgs[++msgIndex];
await receiver.DeadLetterMessageAsync(deadLettered);
var deadLetterScope = _listener.AssertAndRemoveScope(DiagnosticProperty.DeadLetterActivityName);
AssertCommonTags(deadLetterScope.Activity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
var abandoned = receivedMsgs[++msgIndex];
await receiver.AbandonMessageAsync(abandoned);
var abandonScope = _listener.AssertAndRemoveScope(DiagnosticProperty.AbandonActivityName);
AssertCommonTags(abandonScope.Activity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
var receiveDeferMsg = await receiver.ReceiveDeferredMessageAsync(deferred.SequenceNumber);
var receiveDeferScope = _listener.AssertAndRemoveScope(DiagnosticProperty.ReceiveDeferredActivityName);
AssertCommonTags(receiveDeferScope.Activity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
// renew lock
if (useSessions)
{
var sessionReceiver = (ServiceBusSessionReceiver)receiver;
await sessionReceiver.RenewSessionLockAsync();
var renewSessionScope = _listener.AssertAndRemoveScope(DiagnosticProperty.RenewSessionLockActivityName);
AssertCommonTags(renewSessionScope.Activity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
// set state
var state = new BinaryData("state");
await sessionReceiver.SetSessionStateAsync(state);
var setStateScope = _listener.AssertAndRemoveScope(DiagnosticProperty.SetSessionStateActivityName);
AssertCommonTags(setStateScope.Activity, sessionReceiver.EntityPath, sessionReceiver.FullyQualifiedNamespace);
// get state
var getState = await sessionReceiver.GetSessionStateAsync();
Assert.AreEqual(state.ToArray(), getState.ToArray());
var getStateScope = _listener.AssertAndRemoveScope(DiagnosticProperty.GetSessionStateActivityName);
AssertCommonTags(getStateScope.Activity, sessionReceiver.EntityPath, sessionReceiver.FullyQualifiedNamespace);
}
else
{
await receiver.RenewMessageLockAsync(receivedMsgs[4]);
var renewMessageScope = _listener.AssertAndRemoveScope(DiagnosticProperty.RenewMessageLockActivityName);
AssertCommonTags(renewMessageScope.Activity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
}
// schedule
msgs = ServiceBusTestUtilities.GetMessages(numMessages, sessionId);
foreach (var msg in msgs)
{
var seq = await sender.ScheduleMessageAsync(msg, DateTimeOffset.UtcNow.AddMinutes(1));
Assert.IsNotNull(msg.ApplicationProperties[DiagnosticProperty.DiagnosticIdAttribute]);
var messageScope = _listener.AssertAndRemoveScope(DiagnosticProperty.MessageActivityName);
AssertCommonTags(messageScope.Activity, sender.EntityPath, sender.FullyQualifiedNamespace);
var scheduleScope = _listener.AssertAndRemoveScope(DiagnosticProperty.ScheduleActivityName);
AssertCommonTags(scheduleScope.Activity, sender.EntityPath, sender.FullyQualifiedNamespace);
var linkedActivities = scheduleScope.LinkedActivities;
Assert.AreEqual(1, linkedActivities.Count);
Assert.AreEqual(messageScope.Activity.Id, linkedActivities[0].ParentId);
await sender.CancelScheduledMessageAsync(seq);
var cancelScope = _listener.AssertAndRemoveScope(DiagnosticProperty.CancelActivityName);
AssertCommonTags(cancelScope.Activity, sender.EntityPath, sender.FullyQualifiedNamespace);
}
// send a batch
var batch = await sender.CreateMessageBatchAsync();
for (int i = 0; i < numMessages; i++)
{
batch.TryAddMessage(ServiceBusTestUtilities.GetMessage(sessionId));
}
await sender.SendMessagesAsync(batch);
AssertSendActivities(useSessions, sender, batch.AsReadOnly <ServiceBusMessage>());
};
}
public async Task SenderReceiverActivities(bool useSessions)
{
await using (var scope = await ServiceBusScope.CreateWithQueue(enablePartitioning: false, enableSession: useSessions))
{
using var listener = new TestDiagnosticListener(EntityScopeFactory.DiagnosticNamespace);
var client = new ServiceBusClient(TestEnvironment.ServiceBusConnectionString);
ServiceBusSender sender = client.CreateSender(scope.QueueName);
string sessionId = null;
if (useSessions)
{
sessionId = "sessionId";
}
int numMessages = 5;
var msgs = GetMessages(numMessages, sessionId);
await sender.SendMessagesAsync(msgs);
Activity[] sendActivities = AssertSendActivities(useSessions, sender, msgs, listener);
ServiceBusReceiver receiver = null;
if (useSessions)
{
receiver = await client.CreateSessionReceiverAsync(scope.QueueName);
}
else
{
receiver = client.CreateReceiver(scope.QueueName);
}
var remaining = numMessages;
List <ServiceBusReceivedMessage> receivedMsgs = new List <ServiceBusReceivedMessage>();
while (remaining > 0)
{
// loop in case we don't receive all messages in one attempt
var received = await receiver.ReceiveMessagesAsync(remaining);
receivedMsgs.AddRange(received);
(string Key, object Value, DiagnosticListener)receiveStart = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.ReceiveActivityName + ".Start", receiveStart.Key);
Activity receiveActivity = (Activity)receiveStart.Value;
AssertCommonTags(receiveActivity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
CollectionAssert.Contains(
receiveActivity.Tags,
new KeyValuePair <string, string>(
DiagnosticProperty.RequestedMessageCountAttribute,
remaining.ToString()));
CollectionAssert.Contains(
receiveActivity.Tags,
new KeyValuePair <string, string>(
DiagnosticProperty.MessageIdAttribute,
string.Join(",", received.Select(m => m.MessageId).ToArray())));
remaining -= received.Count;
if (useSessions)
{
CollectionAssert.Contains(
receiveActivity.Tags,
new KeyValuePair <string, string>(
DiagnosticProperty.SessionIdAttribute,
string.Join(",", msgs.Select(m => m.SessionId).Distinct().ToArray())));
}
var receiveLinkedActivities = ((IEnumerable <Activity>)receiveActivity.GetType().GetProperty("Links").GetValue(receiveActivity)).ToArray();
for (int i = 0; i < receiveLinkedActivities.Length; i++)
{
Assert.AreEqual(sendActivities[i].ParentId, receiveLinkedActivities[i].ParentId);
}
(string Key, object Value, DiagnosticListener)receiveStop = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.ReceiveActivityName + ".Stop", receiveStop.Key);
}
var msgIndex = 0;
var completed = receivedMsgs[msgIndex];
await receiver.CompleteMessageAsync(completed);
(string Key, object Value, DiagnosticListener)completeStart = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.CompleteActivityName + ".Start", completeStart.Key);
Activity completeActivity = (Activity)completeStart.Value;
AssertCommonTags(completeActivity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
AssertLockTokensTag(completeActivity, completed.LockToken);
(string Key, object Value, DiagnosticListener)completeStop = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.CompleteActivityName + ".Stop", completeStop.Key);
var deferred = receivedMsgs[++msgIndex];
await receiver.DeferMessageAsync(deferred);
(string Key, object Value, DiagnosticListener)deferStart = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.DeferActivityName + ".Start", deferStart.Key);
Activity deferActivity = (Activity)deferStart.Value;
AssertCommonTags(deferActivity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
AssertLockTokensTag(deferActivity, deferred.LockToken);
(string Key, object Value, DiagnosticListener)deferStop = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.DeferActivityName + ".Stop", deferStop.Key);
var deadLettered = receivedMsgs[++msgIndex];
await receiver.DeadLetterMessageAsync(deadLettered);
(string Key, object Value, DiagnosticListener)deadLetterStart = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.DeadLetterActivityName + ".Start", deadLetterStart.Key);
Activity deadLetterActivity = (Activity)deadLetterStart.Value;
AssertCommonTags(deadLetterActivity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
AssertLockTokensTag(deadLetterActivity, deadLettered.LockToken);
(string Key, object Value, DiagnosticListener)deadletterStop = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.DeadLetterActivityName + ".Stop", deadletterStop.Key);
var abandoned = receivedMsgs[++msgIndex];
await receiver.AbandonMessageAsync(abandoned);
(string Key, object Value, DiagnosticListener)abandonStart = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.AbandonActivityName + ".Start", abandonStart.Key);
Activity abandonActivity = (Activity)abandonStart.Value;
AssertCommonTags(abandonActivity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
AssertLockTokensTag(abandonActivity, abandoned.LockToken);
(string Key, object Value, DiagnosticListener)abandonStop = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.AbandonActivityName + ".Stop", abandonStop.Key);
var receiveDeferMsg = await receiver.ReceiveDeferredMessageAsync(deferred.SequenceNumber);
(string Key, object Value, DiagnosticListener)receiveDeferStart = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.ReceiveDeferredActivityName + ".Start", receiveDeferStart.Key);
Activity receiveDeferActivity = (Activity)receiveDeferStart.Value;
AssertCommonTags(receiveDeferActivity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
CollectionAssert.Contains(
receiveDeferActivity.Tags,
new KeyValuePair <string, string>(
DiagnosticProperty.MessageIdAttribute,
deferred.MessageId));
CollectionAssert.Contains(
receiveDeferActivity.Tags,
new KeyValuePair <string, string>(
DiagnosticProperty.SequenceNumbersAttribute,
deferred.SequenceNumber.ToString()));
(string Key, object Value, DiagnosticListener)receiveDeferStop = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.ReceiveDeferredActivityName + ".Stop", receiveDeferStop.Key);
// renew lock
if (useSessions)
{
var sessionReceiver = (ServiceBusSessionReceiver)receiver;
await sessionReceiver.RenewSessionLockAsync();
(string Key, object Value, DiagnosticListener)renewStart = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.RenewSessionLockActivityName + ".Start", renewStart.Key);
Activity renewActivity = (Activity)renewStart.Value;
AssertCommonTags(renewActivity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
CollectionAssert.Contains(
renewActivity.Tags,
new KeyValuePair <string, string>(
DiagnosticProperty.SessionIdAttribute,
"sessionId"));
(string Key, object Value, DiagnosticListener)renewStop = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.RenewSessionLockActivityName + ".Stop", renewStop.Key);
// set state
var state = Encoding.UTF8.GetBytes("state");
await sessionReceiver.SetSessionStateAsync(state);
(string Key, object Value, DiagnosticListener)setStateStart = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.SetSessionStateActivityName + ".Start", setStateStart.Key);
Activity setStateActivity = (Activity)setStateStart.Value;
AssertCommonTags(setStateActivity, sessionReceiver.EntityPath, sessionReceiver.FullyQualifiedNamespace);
CollectionAssert.Contains(
setStateActivity.Tags,
new KeyValuePair <string, string>(
DiagnosticProperty.SessionIdAttribute,
"sessionId"));
(string Key, object Value, DiagnosticListener)setStateStop = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.SetSessionStateActivityName + ".Stop", setStateStop.Key);
// get state
var getState = await sessionReceiver.GetSessionStateAsync();
Assert.AreEqual(state, getState);
(string Key, object Value, DiagnosticListener)getStateStart = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.GetSessionStateActivityName + ".Start", getStateStart.Key);
Activity getStateActivity = (Activity)getStateStart.Value;
AssertCommonTags(getStateActivity, sessionReceiver.EntityPath, sessionReceiver.FullyQualifiedNamespace);
CollectionAssert.Contains(
getStateActivity.Tags,
new KeyValuePair <string, string>(
DiagnosticProperty.SessionIdAttribute,
"sessionId"));
(string Key, object Value, DiagnosticListener)getStateStop = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.GetSessionStateActivityName + ".Stop", getStateStop.Key);
}
else
{
await receiver.RenewMessageLockAsync(receivedMsgs[4]);
(string Key, object Value, DiagnosticListener)renewStart = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.RenewMessageLockActivityName + ".Start", renewStart.Key);
Activity renewActivity = (Activity)renewStart.Value;
AssertCommonTags(renewActivity, receiver.EntityPath, receiver.FullyQualifiedNamespace);
AssertLockTokensTag(renewActivity, receivedMsgs[4].LockToken);
CollectionAssert.Contains(
renewActivity.Tags,
new KeyValuePair <string, string>(
DiagnosticProperty.LockedUntilAttribute,
receivedMsgs[4].LockedUntil.ToString()));
(string Key, object Value, DiagnosticListener)renewStop = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.RenewMessageLockActivityName + ".Stop", renewStop.Key);
}
// schedule
msgs = GetMessages(numMessages, sessionId);
foreach (var msg in msgs)
{
var seq = await sender.ScheduleMessageAsync(msg, DateTimeOffset.UtcNow.AddMinutes(1));
Assert.IsNotNull(msg.Properties[DiagnosticProperty.DiagnosticIdAttribute]);
(string Key, object Value, DiagnosticListener)startMessage = listener.Events.Dequeue();
Activity messageActivity = (Activity)startMessage.Value;
AssertCommonTags(messageActivity, sender.EntityPath, sender.FullyQualifiedNamespace);
Assert.AreEqual(DiagnosticProperty.MessageActivityName + ".Start", startMessage.Key);
(string Key, object Value, DiagnosticListener)stopMessage = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.MessageActivityName + ".Stop", stopMessage.Key);
(string Key, object Value, DiagnosticListener)startSchedule = listener.Events.Dequeue();
AssertCommonTags((Activity)startSchedule.Value, sender.EntityPath, sender.FullyQualifiedNamespace);
Assert.AreEqual(DiagnosticProperty.ScheduleActivityName + ".Start", startSchedule.Key);
(string Key, object Value, DiagnosticListener)stopSchedule = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.ScheduleActivityName + ".Stop", stopSchedule.Key);
var linkedActivities = ((IEnumerable <Activity>)startSchedule.Value.GetType().GetProperty("Links").GetValue(startSchedule.Value)).ToArray();
Assert.AreEqual(1, linkedActivities.Length);
Assert.AreEqual(messageActivity.Id, linkedActivities[0].ParentId);
await sender.CancelScheduledMessageAsync(seq);
(string Key, object Value, DiagnosticListener)startCancel = listener.Events.Dequeue();
AssertCommonTags((Activity)startCancel.Value, sender.EntityPath, sender.FullyQualifiedNamespace);
Assert.AreEqual(DiagnosticProperty.CancelActivityName + ".Start", startCancel.Key);
(string Key, object Value, DiagnosticListener)stopCancel = listener.Events.Dequeue();
Assert.AreEqual(DiagnosticProperty.CancelActivityName + ".Stop", stopCancel.Key);
}
// send a batch
var batch = await sender.CreateMessageBatchAsync();
for (int i = 0; i < numMessages; i++)
{
batch.TryAddMessage(GetMessage(sessionId));
}
await sender.SendMessagesAsync(batch);
AssertSendActivities(useSessions, sender, batch.AsEnumerable <ServiceBusMessage>(), listener);
};
}
private async Task RunBatchReceiveLoopAsync(CancellationToken cancellationToken)
{
ServiceBusClient sessionClient = null;
ServiceBusReceiver receiver = null;
if (_isSessionsEnabled)
{
sessionClient = _client.Value;
}
else
{
receiver = _batchReceiver.Value;
}
while (true)
{
try
{
if (cancellationToken.IsCancellationRequested)
{
_logger.LogInformation("Message processing has been stopped or cancelled");
return;
}
if (_isSessionsEnabled && (receiver == null || receiver.IsClosed))
{
try
{
receiver = await sessionClient.AcceptNextSessionAsync(
_entityPath,
new ServiceBusSessionReceiverOptions
{
PrefetchCount = _serviceBusOptions.PrefetchCount
},
cancellationToken).ConfigureAwait(false);
}
catch (ServiceBusException ex)
when(ex.Reason == ServiceBusFailureReason.ServiceTimeout)
{
// it's expected if the entity is empty, try next time
continue;
}
}
IReadOnlyList <ServiceBusReceivedMessage> messages =
await receiver.ReceiveMessagesAsync(
_serviceBusOptions.MaxMessageBatchSize,
cancellationToken : cancellationToken).AwaitWithCancellation(cancellationToken);
if (messages.Count > 0)
{
ServiceBusReceivedMessage[] messagesArray = messages.ToArray();
ServiceBusTriggerInput input = ServiceBusTriggerInput.CreateBatch(messagesArray);
if (_isSessionsEnabled)
{
input.MessageActions = new ServiceBusSessionMessageActions((ServiceBusSessionReceiver)receiver);
}
else
{
input.MessageActions = new ServiceBusMessageActions(receiver);
}
FunctionResult result = await _triggerExecutor.TryExecuteAsync(input.GetTriggerFunctionData(), cancellationToken).ConfigureAwait(false);
// Complete batch of messages only if the execution was successful
if (_autoCompleteMessagesOptionEvaluatedValue)
{
if (result.Succeeded)
{
List <Task> completeTasks = new List <Task>();
foreach (ServiceBusReceivedMessage message in messagesArray)
{
// skip messages that were settled in the user's function
if (input.MessageActions.SettledMessages.Contains(message))
{
continue;
}
// Pass CancellationToken.None to allow autocompletion to finish even when shutting down
completeTasks.Add(receiver.CompleteMessageAsync(message, CancellationToken.None));
}
await Task.WhenAll(completeTasks).ConfigureAwait(false);
}
else
{
List <Task> abandonTasks = new List <Task>();
foreach (ServiceBusReceivedMessage message in messagesArray)
{
// skip messages that were settled in the user's function
if (input.MessageActions.SettledMessages.Contains(message))
{
continue;
}
// Pass CancellationToken.None to allow abandon to finish even when shutting down
abandonTasks.Add(receiver.AbandonMessageAsync(message, cancellationToken: CancellationToken.None));
}
await Task.WhenAll(abandonTasks).ConfigureAwait(false);
}
}
}
else
{
// Close the session and release the session lock after draining all messages for the accepted session.
if (_isSessionsEnabled)
{
// Use CancellationToken.None to attempt to close the receiver even when shutting down
await receiver.CloseAsync(CancellationToken.None).ConfigureAwait(false);
}
}
}
catch (ObjectDisposedException)
{
// Ignore as we are stopping the host
}
catch (OperationCanceledException)
when(cancellationToken.IsCancellationRequested)
{
// Ignore as we are stopping the host
_logger.LogInformation("Message processing has been stopped or cancelled");
}
catch (Exception ex)
{
// Log another exception
_logger.LogError(ex, $"An unhandled exception occurred in the message batch receive loop");
if (_isSessionsEnabled && receiver != null)
{
// Attempt to close the session and release session lock to accept a new session on the next loop iteration
try
{
// Use CancellationToken.None to attempt to close the receiver even when shutting down
await receiver.CloseAsync(CancellationToken.None).ConfigureAwait(false);
}
catch
{
// Best effort
receiver = null;
}
}
}
}
}
public override Task AbandonMessageAsync(ServiceBusReceivedMessage message, IDictionary <string, object> propertiesToModify = null, CancellationToken cancellationToken = default)
{
return(serviceBusReceiver.AbandonMessageAsync(message, propertiesToModify, cancellationToken));
}