private Task CloseSession(bool success, IMessageSession session, CancellationToken cancellationToken)
{
return(receiveRetryPolicy.ExecuteAsync(
async() =>
{
try
{
await session.CloseAsync();
if (success)
{
dynamicThrottling.NotifyWorkCompleted();
}
else
{
dynamicThrottling.NotifyWorkCompletedWithError();
}
return Task.CompletedTask;
}
catch (Exception ex)
{
logger.LogError($"An unrecoverable error occurred while trying to CLOSE a session in subscription \"{subscription}\":\r\n{ex.Message}");
dynamicThrottling.NotifyWorkCompletedWithError();
return Task.CompletedTask;
}
}));
}
private void ProcessPartition(string key)
{
//this.instrumentation.EventPublisherStarted();
this.queue.GetPendingAsync(
key,
(results, hasMoreResults) =>
{
var enumerator = results.GetEnumerator();
this.SendAndDeletePending(
enumerator,
allElementWereProcessed =>
{
enumerator.Dispose();
if (!allElementWereProcessed)
{
this.EnqueueIfNotExists(key);
}
else if (hasMoreResults)
{
// if there are more events in this partition, then continue processing and do not mark work as completed.
ProcessPartition(key);
return;
}
// all elements were processed or should be retried later. Mark this job as done.
this.dynamicThrottling.NotifyWorkCompleted();
//this.instrumentation.EventPublisherFinished();
},
ex =>
{
enumerator.Dispose();
logger.LogError("An error occurred while publishing events for partition {0}:\r\n{1}", key, ex);
// if there was ANY unhandled error, re-add the item to collection.
EnqueueIfNotExists(key);
dynamicThrottling.NotifyWorkCompletedWithError();
//instrumentation.EventPublisherFinished();
});
},
ex =>
{
logger.LogError("An error occurred while getting the events pending for publishing for partition {0}:\r\n{1}", key, ex);
// if there was ANY unhandled error, re-add the item to collection.
EnqueueIfNotExists(key);
dynamicThrottling.NotifyWorkCompletedWithError();
//instrumentation.EventPublisherFinished();
});
}
private async Task ReleaseMessage(Message msg, MessageReleaseAction releaseAction, Stopwatch roundtripStopwatch)
{
switch (releaseAction.Kind)
{
case MessageReleaseActionKind.Complete:
await msg.SafeCompleteAsync(
this.subscription,
client,
success =>
{
if (success)
{
this.dynamicThrottling.NotifyWorkCompleted();
}
else
{
this.dynamicThrottling.NotifyWorkCompletedWithError();
}
},
logger,
roundtripStopwatch);
break;
case MessageReleaseActionKind.Abandon:
await msg.SafeAbandonAsync(
this.subscription,
client,
success => {
dynamicThrottling.NotifyWorkCompletedWithError();
},
logger,
roundtripStopwatch);
break;
case MessageReleaseActionKind.DeadLetter:
await msg.SafeDeadLetterAsync(
this.subscription,
client,
releaseAction.DeadLetterReason,
releaseAction.DeadLetterDescription,
success => {
this.dynamicThrottling.NotifyWorkCompletedWithError();
},
logger,
roundtripStopwatch);
break;
default:
break;
}
}
public void penalize_decreases_less_than_completed_with_error()
{
using (var sut1 = new DynamicThrottling(100, 10, 3, 5, 1, 8000))
using (var sut2 = new DynamicThrottling(100, 10, 3, 5, 1, 8000))
{
IncreaseDegreesOfParallelism(sut1);
IncreaseDegreesOfParallelism(sut2);
sut1.NotifyWorkStarted();
sut2.NotifyWorkStarted();
sut1.Penalize();
sut2.NotifyWorkCompletedWithError();
Assert.True(sut1.AvailableDegreesOfParallelism > sut2.AvailableDegreesOfParallelism);
}
}
public void penalize_decreases_less_than_completed_with_error()
{
using (var sut1 = new DynamicThrottling(100, 10, 3, 5, 1, 8000))
using (var sut2 = new DynamicThrottling(100, 10, 3, 5, 1, 8000))
{
IncreaseDegreesOfParallelism(sut1);
IncreaseDegreesOfParallelism(sut2);
sut1.NotifyWorkStarted();
sut2.NotifyWorkStarted();
sut1.Penalize();
sut2.NotifyWorkCompletedWithError();
Assert.True(sut1.AvailableDegreesOfParallelism > sut2.AvailableDegreesOfParallelism);
}
}
/// <summary>
/// Receives the messages in an asynchronous loop and closes the session once there are no more messages.
/// </summary>
private void ReceiveMessagesAndCloseSession(MessageSession session, CancellationToken cancellationToken)
{
var unreleasedMessages = new CountdownEvent(1);
Action <bool> closeSession = success => {
Action doClose = () => {
try {
unreleasedMessages.Signal();
if (!unreleasedMessages.Wait(15000, cancellationToken))
{
Trace.TraceWarning("Waited for pending unreleased messages before closing session in subscription {0} but they did not complete in time", subscription);
}
} catch (OperationCanceledException) { } finally {
unreleasedMessages.Dispose();
}
receiveRetryPolicy.ExecuteAction(
cb => session.BeginClose(cb, null),
session.EndClose,
() => {
instrumentation.SessionEnded();
if (success)
{
dynamicThrottling.NotifyWorkCompleted();
}
else
{
dynamicThrottling.NotifyWorkCompletedWithError();
}
},
ex => {
instrumentation.SessionEnded();
Trace.TraceError("An unrecoverable error occurred while trying to close a session in subscription {1}:\r\n{0}", ex, subscription);
dynamicThrottling.NotifyWorkCompletedWithError();
});
};
if (requiresSequentialProcessing)
{
doClose.Invoke();
}
else
{
// Allow some time for releasing the messages before closing. Also, continue in a non I/O completion thread in order to block.
TaskEx.Delay(200).ContinueWith(t => doClose());
}
};
// Declare an action to receive the next message in the queue or closes the session if cancelled.
Action receiveNext = null;
// Declare an action acting as a callback whenever a non-transient exception occurs while receiving or processing messages.
Action <Exception> recoverReceive = null;
// Declare an action responsible for the core operations in the message receive loop.
Action receiveMessage = () => {
// Use a retry policy to execute the Receive action in an asynchronous and reliable fashion.
receiveRetryPolicy.ExecuteAction
(
cb => {
// Start receiving a new message asynchronously.
// Does not wait for new messages to arrive in a session. If no further messages we will just close the session.
session.BeginReceive(TimeSpan.Zero, cb, null);
},
// Complete the asynchronous operation. This may throw an exception that will be handled internally by retry policy.
session.EndReceive,
msg => {
// Process the message once it was successfully received
// Check if we actually received any messages.
if (msg != null)
{
var roundtripStopwatch = Stopwatch.StartNew();
long schedulingElapsedMilliseconds = 0;
long processingElapsedMilliseconds = 0;
unreleasedMessages.AddCount();
Task.Factory.StartNew(() => {
var releaseAction = MessageReleaseAction.AbandonMessage;
try {
instrumentation.MessageReceived();
schedulingElapsedMilliseconds = roundtripStopwatch.ElapsedMilliseconds;
// Make sure the process was told to stop receiving while it was waiting for a new message.
if (!cancellationToken.IsCancellationRequested)
{
try {
try {
// Process the received message.
releaseAction = InvokeMessageHandler(msg);
processingElapsedMilliseconds = roundtripStopwatch.ElapsedMilliseconds - schedulingElapsedMilliseconds;
instrumentation.MessageProcessed(releaseAction.Kind == MessageReleaseActionKind.Complete, processingElapsedMilliseconds);
} catch {
processingElapsedMilliseconds = roundtripStopwatch.ElapsedMilliseconds - schedulingElapsedMilliseconds;
instrumentation.MessageProcessed(false, processingElapsedMilliseconds);
throw;
}
} finally {
if (roundtripStopwatch.Elapsed > TimeSpan.FromSeconds(45))
{
dynamicThrottling.Penalize();
}
}
}
} finally {
// Ensure that any resources allocated by a BrokeredMessage instance are released.
if (requiresSequentialProcessing)
{
ReleaseMessage(msg, releaseAction, () => { receiveNext(); }, () => { closeSession(false); }, unreleasedMessages, processingElapsedMilliseconds,
schedulingElapsedMilliseconds, roundtripStopwatch);
}
else
{
// Receives next without waiting for the message to be released.
ReleaseMessage(msg, releaseAction, () => { }, () => { dynamicThrottling.Penalize(); }, unreleasedMessages, processingElapsedMilliseconds,
schedulingElapsedMilliseconds, roundtripStopwatch);
receiveNext.Invoke();
}
}
});
}
else
{
// no more messages in the session, close it and do not continue receiving
closeSession(true);
}
},
ex => {
// Invoke a custom action to indicate that we have encountered an exception and
// need further decision as to whether to continue receiving messages.
recoverReceive.Invoke(ex);
});
};
// Initialize an action to receive the next message in the queue or closes the session if cancelled.
receiveNext = () => {
if (!cancellationToken.IsCancellationRequested)
{
// Continue receiving and processing new messages until told to stop.
receiveMessage.Invoke();
}
else
{
closeSession(true);
}
};
// Initialize a custom action acting as a callback whenever a non-transient exception occurs while receiving or processing messages.
recoverReceive = ex => {
// Just log an exception. Do not allow an unhandled exception to terminate the message receive loop abnormally.
Trace.TraceError("An unrecoverable error occurred while trying to receive a new message from subscription {1}:\r\n{0}", ex, subscription);
// Cannot continue to receive messages from this session.
closeSession(false);
};
// Start receiving messages asynchronously for the session.
receiveNext.Invoke();
}
/// <summary>
/// Receives the messages in an endless asynchronous loop.
/// </summary>
private void ReceiveMessages(CancellationToken cancellationToken)
{
// Declare an action to receive the next message in the queue or end if cancelled.
Action receiveNext = null;
// Declare an action acting as a callback whenever a non-transient exception occurs while receiving or processing messages.
Action <Exception> recoverReceive = null;
// Declare an action responsible for the core operations in the message receive loop.
Action receiveMessage = () => {
// Use a retry policy to execute the Receive action in an asynchronous and reliable fashion.
receiveRetryPolicy.ExecuteAction
(
cb => {
// Start receiving a new message asynchronously.
client.BeginReceive(ReceiveLongPollingTimeout, cb, null);
},
ar => {
// Complete the asynchronous operation. This may throw an exception that will be handled internally by retry policy.
try {
return(client.EndReceive(ar));
} catch (TimeoutException) {
// TimeoutException is not just transient but completely expected in this case, so not relying on Topaz to retry
return(null);
}
},
msg => {
// Process the message once it was successfully received
if (processInParallel)
{
// Continue receiving and processing new messages asynchronously
Task.Factory.StartNew(receiveNext);
}
// Check if we actually received any messages.
if (msg != null)
{
var roundtripStopwatch = Stopwatch.StartNew();
long schedulingElapsedMilliseconds = 0;
long processingElapsedMilliseconds = 0;
Task.Factory.StartNew(() => {
var releaseAction = MessageReleaseAction.AbandonMessage;
try {
instrumentation.MessageReceived();
schedulingElapsedMilliseconds = roundtripStopwatch.ElapsedMilliseconds;
// Make sure the process was told to stop receiving while it was waiting for a new message.
if (!cancellationToken.IsCancellationRequested)
{
try {
try {
// Process the received message.
releaseAction = InvokeMessageHandler(msg);
processingElapsedMilliseconds = roundtripStopwatch.ElapsedMilliseconds - schedulingElapsedMilliseconds;
instrumentation.MessageProcessed(releaseAction.Kind == MessageReleaseActionKind.Complete, processingElapsedMilliseconds);
} catch {
processingElapsedMilliseconds = roundtripStopwatch.ElapsedMilliseconds - schedulingElapsedMilliseconds;
instrumentation.MessageProcessed(false, processingElapsedMilliseconds);
throw;
}
} finally {
if (roundtripStopwatch.Elapsed > TimeSpan.FromSeconds(45))
{
dynamicThrottling.Penalize();
}
}
}
} finally {
// Ensure that any resources allocated by a BrokeredMessage instance are released.
ReleaseMessage(msg, releaseAction, processingElapsedMilliseconds, schedulingElapsedMilliseconds, roundtripStopwatch);
}
if (!processInParallel)
{
// Continue receiving and processing new messages until told to stop.
receiveNext.Invoke();
}
});
}
else
{
dynamicThrottling.NotifyWorkCompleted();
if (!processInParallel)
{
// Continue receiving and processing new messages until told to stop.
receiveNext.Invoke();
}
}
},
ex => {
// Invoke a custom action to indicate that we have encountered an exception and
// need further decision as to whether to continue receiving messages.
recoverReceive.Invoke(ex);
});
};
// Initialize an action to receive the next message in the queue or end if cancelled.
receiveNext = () => {
dynamicThrottling.WaitUntilAllowedParallelism(cancellationToken);
if (!cancellationToken.IsCancellationRequested)
{
dynamicThrottling.NotifyWorkStarted();
// Continue receiving and processing new messages until told to stop.
receiveMessage.Invoke();
}
};
// Initialize a custom action acting as a callback whenever a non-transient exception occurs while receiving or processing messages.
recoverReceive = ex => {
// Just log an exception. Do not allow an unhandled exception to terminate the message receive loop abnormally.
Trace.TraceError("An unrecoverable error occurred while trying to receive a new message from subscription {1}:\r\n{0}", ex, subscription);
dynamicThrottling.NotifyWorkCompletedWithError();
if (!cancellationToken.IsCancellationRequested)
{
// Continue receiving and processing new messages until told to stop regardless of any exceptions.
TaskEx.Delay(10000).ContinueWith(t => receiveMessage.Invoke());
}
};
// Start receiving messages asynchronously.
receiveNext.Invoke();
}
