public void TestSerial_NormalProcessing()
{
// Arrange
const int runCount = 100;
var client = new CloudEnvironment();
var overflow = client.BlobClient.GetContainerReference("overflownqueues-1");
var queue = client.QueueClient.GetQueueReference("test1");
var result = string.Empty;
var expected = string.Empty;
var sw = new Stopwatch();
var factory = new AzureExtendedQueueFactory(new AzureBlobContainer(overflow), new ConsoleLogService());
var equeue = factory.Create(new AzureQueue(queue));
for (var i = 0; i < runCount; i++)
{
expected += i.ToString(CultureInfo.InvariantCulture);
}
using (var mre = new ManualResetEvent(false))
{
var options = new HandleMessagesSerialOptions(
TimeSpan.FromSeconds(0),
TimeSpan.FromMinutes(2),
TimeSpan.FromSeconds(30),
5,
new CancellationToken(),
message =>
{
if (message.GetMessageContents <string>() == "END")
{
mre.Set();
return(Task.FromResult(true));
}
result += message.GetMessageContents <string>();
return(Task.FromResult(true));
},
null,
ex => { throw ex; });
// Act
sw.Start();
queue.CreateIfNotExists();
overflow.CreateIfNotExists();
queue.Clear();
for (var i = 0; i < runCount; i++)
{
equeue.AddMessageEntity(i.ToString(CultureInfo.InvariantCulture));
}
equeue.AddMessageEntity("END");
equeue.HandleMessagesInSerialAsync(options);
// Assert
mre.WaitOne();
sw.Stop();
Trace.WriteLine("Total execution time (in seconds): " + sw.Elapsed.TotalSeconds.ToString(CultureInfo.InvariantCulture));
Trace.WriteLine(equeue.Statistics);
Assert.AreEqual(expected, result);
}
}
public void TestErrorDuringRetrieval_GetOverflownContents()
{
// Arrange
var client = new CloudEnvironment();
var queue = client.QueueClient.GetQueueReference("test7");
var overflow = client.BlobClient.GetContainerReference("overflownqueues-7");
var rnd = new Random();
ComplexModel result = null;
var expected = new ComplexModel {
Name = new string(Enumerable.Range(1, 128 * 1024).Select(r => (char)rnd.Next(1024, 4096)).ToArray())
};
var sw = new Stopwatch();
var succeeded = false;
var factoryOne = new DefaultExtendedQueueFactory(
new AzureQueueMessageProvider(),
new AzureMaximumMessageSizeProvider(),
new AzureMaximumMessagesPerRequestProvider(),
new ChaosMonkeyOverflownMessageHandler(new AzureBlobContainer(overflow), ChaosMonkeyOverflownMessageHandler.FailureMode.GetOverflownContents),
new ConsoleLogService()
);
new AzureExtendedQueueFactory(new AzureBlobContainer(overflow), new ConsoleLogService());
var equeue = factoryOne.Create(new AzureQueue(queue));
using (var mre = new ManualResetEvent(false))
{
var options = new HandleMessagesSerialOptions(
TimeSpan.FromSeconds(0),
TimeSpan.FromMinutes(2),
TimeSpan.FromSeconds(30),
1,
new CancellationToken(),
message =>
{
result = message.GetMessageContents <ComplexModel>();
mre.Set();
return(Task.FromResult(true));
},
null,
ex =>
{
succeeded = true;
mre.Set();
});
// Act
sw.Start();
queue.CreateIfNotExists();
overflow.CreateIfNotExists();
queue.Clear();
equeue.AddMessageEntity(expected);
equeue.HandleMessagesInSerialAsync(options);
// Assert
mre.WaitOne();
sw.Stop();
Trace.WriteLine("Total execution time (in seconds): " + sw.Elapsed.TotalSeconds.ToString(CultureInfo.InvariantCulture));
Assert.IsTrue(succeeded);
}
}
/// <summary>
/// Handles poison messages by either delegating it to a handler or deleting it if no handler is provided.
/// </summary>
/// <param name="message">The message to operate on.</param>
/// <param name="messageOptions">Initialisation options for this method.</param>
/// <param name="asyncLock">An object that's responsible for synchronising access to shared resources in an asynchronous manner.</param>
/// <param name="messageSpecificCancellationTokenSource">The message-specific cancellation token.</param>
/// <returns>
/// True if the <paramref name="message" /> was deleted; <see langword="false" /> if it should be requeued and <see langword="checked" /> again.
/// </returns>
private async Task <bool> WasPoisonMessageAndRemoved(
[NotNull] HandleMessagesSerialOptions messageOptions,
[NotNull] QueueMessageWrapper message,
[NotNull] AsyncLock asyncLock,
[NotNull] CancellationTokenSource messageSpecificCancellationTokenSource)
{
Guard.NotNull(messageOptions, "messageOptions");
Guard.NotNull(message, "message");
Guard.NotNull(asyncLock, "asyncLock");
Guard.NotNull(messageSpecificCancellationTokenSource, "messageSpecificCancellationTokenSource");
if (message.ActualMessage.DequeueCount <= messageOptions.PoisonMessageThreshold)
{
return(false);
}
if (messageOptions.PoisonHandler != null && !(await messageOptions.PoisonHandler(message).ConfigureAwait(false)))
{
return(false);
}
this.Statistics.IncreasePoisonMessages();
await this.Top.SyncDeleteMessage(asyncLock, message, messageSpecificCancellationTokenSource).ConfigureAwait(false);
return(true);
}
protected internal override void SerialFinallyHandler(
HandleMessagesSerialOptions messageOptions,
Task keepAliveTask,
IQueueMessage message,
CancellationTokenSource messageSpecificCancellationTokenSource)
{
this.DecoratedQueue.SerialFinallyHandler(messageOptions, keepAliveTask, message, messageSpecificCancellationTokenSource);
}
/// <summary>
/// Error handler for cancelled task exception during serial message processing.
/// </summary>
/// <param name="messageOptions">The message options object.</param>
protected internal virtual bool HandleTaskCancelled(HandleMessagesSerialOptions messageOptions)
{
this.Top.LogAction(LogSeverity.Info, "Attempting to cancel No messages found when an attempt was made to read from a queue", "Queue: {0}", this.Name);
if (messageOptions.CancelToken.IsCancellationRequested)
{
return(true);
}
return(false);
}
public void TestSerializedMessages()
{
// Arrange
var client = new CloudEnvironment();
var queue = client.QueueClient.GetQueueReference("test5");
var overflow = client.BlobClient.GetContainerReference("overflownqueues-5");
ComplexModel result = null;
var expected = new ComplexModel {
Name = "Test"
};
var sw = new Stopwatch();
var factory = new AzureExtendedQueueFactory(new AzureBlobContainer(overflow), new ConsoleLogService());
var equeue = factory.Create(new AzureQueue(queue));
using (var mre = new ManualResetEvent(false))
{
var options = new HandleMessagesSerialOptions(
TimeSpan.FromSeconds(0),
TimeSpan.FromMinutes(2),
TimeSpan.FromSeconds(30),
5,
new CancellationToken(),
message =>
{
result = message.GetMessageContents <ComplexModel>();
mre.Set();
return(Task.FromResult(true));
},
null,
ex => { throw ex; });
// Act
sw.Start();
queue.CreateIfNotExists();
overflow.CreateIfNotExists();
queue.Clear();
equeue.AddMessageEntity(expected);
equeue.HandleMessagesInSerialAsync(options);
// Assert
mre.WaitOne();
sw.Stop();
Trace.WriteLine("Total execution time (in seconds): " + sw.Elapsed.TotalSeconds.ToString(CultureInfo.InvariantCulture));
Assert.AreEqual(expected.Name, result.Name);
Assert.AreEqual(expected.ADictionary.First().Key, result.ADictionary.First().Key);
Assert.AreEqual(expected.ADictionary.First().Value, result.ADictionary.First().Value);
Assert.AreEqual(expected.AList.First(), result.AList.First());
}
}
/// <summary>
/// Processes a queue message.
/// </summary>
/// <param name="message">The message to be processed.</param>
/// <param name="messageOptions">Initialisation options for the method that handles the messages.</param>
/// <param name="messageSpecificCancellationTokenSource">A cancellation token source that's specific to this message.</param>
/// <param name="asyncLock">An object that's responsible for synchronising access to shared resources in an asynchronous manner.</param>
private async Task <Task> ProcessMessageInternal(
[NotNull] QueueMessageWrapper message,
[NotNull] HandleMessagesSerialOptions messageOptions,
[NotNull] CancellationTokenSource messageSpecificCancellationTokenSource)
{
Guard.NotNull(message, "message");
Guard.NotNull(messageOptions, "messageOptions");
Guard.NotNull(messageSpecificCancellationTokenSource, "messageSpecificCancellationTokenSource");
var asynclock = new AsyncLock();
// Very old message; delete it and move to the next one
if (messageOptions.TimeWindow.TotalSeconds > 0 &&
(!message.ActualMessage.InsertionTime.HasValue || message.ActualMessage.InsertionTime.Value.UtcDateTime.Add(messageOptions.TimeWindow) < DateTime.UtcNow))
{
await this.Top.SyncDeleteMessage(asynclock, message, messageSpecificCancellationTokenSource).ConfigureAwait(false);
return(Task.FromResult <Task>(null));
}
// Handles poison messages by either delegating it to a handler or deleting it if no handler is provided.
if (await this.Top.WasPoisonMessageAndRemoved(messageOptions, message, asynclock, messageSpecificCancellationTokenSource).ConfigureAwait(false))
{
return(Task.FromResult <Task>(null));
}
// Starts the background thread which ensures message leases stay fresh.
var keepAliveTask = this.KeepMessageAlive(message.ActualMessage, messageOptions.MessageLeaseTime, messageSpecificCancellationTokenSource.Token, asynclock);
using (var comboCancelToken = CancellationTokenSource.CreateLinkedTokenSource(messageSpecificCancellationTokenSource.Token, messageOptions.CancelToken))
{
// Execute the provided action and if successful, delete the message.
if (await messageOptions.MessageHandler(message).ConfigureAwait(false))
{
this.Statistics.IncreaseSuccessfulMessages();
await this.Top.SyncDeleteMessage(asynclock, message, comboCancelToken).ConfigureAwait(false);
}
else
{
this.Statistics.IncreaseReenqueuesCount();
}
}
messageSpecificCancellationTokenSource.Cancel();
return(keepAliveTask);
}
public void TestOverflownMessages()
{
// Arrange
var client = new CloudEnvironment();
var queue = client.QueueClient.GetQueueReference("test4");
var overflow = client.BlobClient.GetContainerReference("overflownqueues-4");
var result = string.Empty;
var rnd = new Random();
var expected = new string(Enumerable.Range(1, 128 * 1024).Select(r => (char)rnd.Next(1024, 4096)).ToArray());
var sw = new Stopwatch();
var factory = new AzureExtendedQueueFactory(new AzureBlobContainer(overflow), new ConsoleLogService());
var equeue = factory.Create(new AzureQueue(queue));
using (var mre = new ManualResetEvent(false))
{
var options = new HandleMessagesSerialOptions(
TimeSpan.FromSeconds(0),
TimeSpan.FromMinutes(2),
TimeSpan.FromSeconds(30),
5,
new CancellationToken(),
message =>
{
result = message.GetMessageContents <string>();
mre.Set();
return(Task.FromResult(true));
},
null,
ex => { throw ex; });
// Act
sw.Start();
queue.CreateIfNotExists();
overflow.CreateIfNotExists();
queue.Clear();
equeue.AddMessageEntity(expected);
equeue.HandleMessagesInSerialAsync(options);
// Assert
mre.WaitOne();
sw.Stop();
Trace.WriteLine("Total execution time (in seconds): " + sw.Elapsed.TotalSeconds.ToString(CultureInfo.InvariantCulture));
Assert.AreEqual(expected, result);
}
}
/// <summary>
/// The finally handler in the try/catch/finally statement of HandleMessagesAsync.
/// </summary>
/// <param name="messageOptions">The message options object.</param>
/// <param name="keepAliveTask">The <see cref="Task" /> that keeps the message "alive".</param>
/// <param name="message">The message currently processing.</param>
/// <param name="messageSpecificCancellationTokenSource">The cancellation token for the message.</param>
protected internal virtual void SerialFinallyHandler(
[CanBeNull] HandleMessagesSerialOptions messageOptions,
[CanBeNull] Task keepAliveTask,
[CanBeNull] IQueueMessage message,
[NotNull] CancellationTokenSource messageSpecificCancellationTokenSource)
{
if (Guard.IsAnyNull(messageOptions, messageSpecificCancellationTokenSource))
{
return;
}
// Cancel any outstanding jobs due to the faulted operation (the keepalive task should have been cancelled)
if (keepAliveTask != null && !keepAliveTask.IsCompleted)
{
if (message != null)
{
this.Top.LogAction(LogSeverity.Warning, "Message was not processed successfully", "Queue's '{0}' message '{1}', processing faulted; cancelling related jobs", this.Name, message.Id);
messageSpecificCancellationTokenSource.Cancel();
}
}
}
public void TestSerial_SerialProcessingDelayed()
{
// Arrange
const int runCount = 3;
var client = new CloudEnvironment();
var queue = client.QueueClient.GetQueueReference("test10");
var overflow = client.BlobClient.GetContainerReference("overflownqueues-10");
var locking = new object();
var result = string.Empty;
var expected = string.Empty;
var sw = new Stopwatch();
long actuallyRun = 0;
var factory = new AzureExtendedQueueFactory(new AzureBlobContainer(overflow), new ConsoleLogService());
var equeue = factory.Create(new AzureQueue(queue));
var lck = new AsyncLock();
for (var i = 1; i < runCount + 1; i++)
{
expected += ((char)(i)).ToString(CultureInfo.InvariantCulture);
}
using (var mre = new ManualResetEvent(false))
{
var options = new HandleMessagesSerialOptions(
TimeSpan.FromSeconds(0),
TimeSpan.FromSeconds(30),
TimeSpan.FromSeconds(30),
5,
new CancellationToken(),
async message =>
{
using (await lck.LockAsync())
{
var innersw = new Stopwatch();
innersw.Start();
// Intentional spinning
while (true)
{
if (innersw.Elapsed > TimeSpan.FromSeconds(33))
{
break;
}
}
var character = message.GetMessageContents <string>();
result += character;
}
if (Interlocked.Increment(ref actuallyRun) == runCount)
{
mre.Set();
}
return(true);
},
null,
ex => { throw ex; });
// Act
sw.Start();
queue.CreateIfNotExists();
overflow.CreateIfNotExists();
queue.Clear();
for (var i = 1; i < runCount + 1; i++)
{
equeue.AddMessageEntity(((char)(i)).ToString(CultureInfo.InvariantCulture));
}
equeue.HandleMessagesInSerialAsync(options);
// Assert
mre.WaitOne();
sw.Stop();
Trace.WriteLine("Total execution time (in seconds): " + sw.Elapsed.TotalSeconds.ToString(CultureInfo.InvariantCulture));
Assert.IsTrue(expected.All(c => result.Contains(c)));
}
}
protected internal override bool HandleTaskCancelled(HandleMessagesSerialOptions messageOptions)
{
return(this.DecoratedQueue.HandleTaskCancelled(messageOptions));
}
protected internal override Task <IQueueMessage> GetMessageFromQueue(HandleMessagesSerialOptions messageOptions, CancellationTokenSource messageSpecificCancellationTokenSource)
{
return(this.DecoratedQueue.GetMessageFromQueue(messageOptions, messageSpecificCancellationTokenSource));
}
/// <summary>
/// Begins a task that receives messages serially and automatically manages their lifetime.
/// </summary>
/// <param name="messageOptions">An options object used to initialise the procedure.</param>
/// <param name="invoker">The (optional) decorator that called this method.</param>
/// <returns>
/// A cancellable task representing the message processing procedure.
/// </returns>
public async Task HandleMessagesInSerialAsync(HandleMessagesSerialOptions messageOptions)
{
Guard.NotNull(messageOptions, "messageOptions");
this.Statistics.IncreaseListeners();
this.Statistics.IncreaseAllMessageSlots();
while (true)
{
if (messageOptions.CancelToken.IsCancellationRequested)
{
this.Statistics.DecreaseListeners();
this.Statistics.DecreaseAllMessageSlots();
return;
}
// When set to true, the queue won't wait before it requests another message
var receivedMessage = false;
// Used to prevent a message operation from running on a specific message
var messageSpecificCancellationTokenSource = new CancellationTokenSource();
Task keepAliveTask = null;
IQueueMessage message = null;
try
{
this.Top.LogAction(LogSeverity.Debug, "Attempting to read from a queue", "Queue: {0}", this.Name);
message = await this.Top.GetMessageFromQueue(messageOptions, messageSpecificCancellationTokenSource).ConfigureAwait(false);
if (message != null)
{
this.Statistics.IncreaseBusyMessageSlots();
this.Top.LogAction(LogSeverity.Debug, "One message found in the queue and will be processed", "Queue: {0}, Message ID: {1}", this.Name, message.Id);
receivedMessage = true;
keepAliveTask = await this.Top.ProcessMessageInternal(
new QueueMessageWrapper(this.Top, message),
messageOptions,
messageSpecificCancellationTokenSource).ConfigureAwait(false);
}
else
{
this.Top.LogAction(LogSeverity.Debug, "No messages found when an attempt was made to read from a queue", "Queue: {0}", this.Name);
}
}
catch (TaskCanceledException)
{
if (this.Top.HandleTaskCancelled(messageOptions))
{
this.Statistics.DecreaseListeners();
this.Statistics.DecreaseAllMessageSlots();
this.Statistics.DecreaseBusyMessageSlots();
return;
}
}
catch (CloudToolsStorageException ex)
{
this.Top.HandleStorageExceptions(messageOptions, ex);
}
catch (Exception ex)
{
this.Top.HandleGeneralExceptions(messageOptions, ex);
}
finally
{
this.Top.SerialFinallyHandler(messageOptions, keepAliveTask, message, messageSpecificCancellationTokenSource);
}
this.Statistics.DecreaseBusyMessageSlots();
// Delay the next polling attempt for a new message, since no messages were received last time.
if (!receivedMessage)
{
await Task.Delay(messageOptions.PollFrequency, messageOptions.CancelToken).ConfigureAwait(false);
}
}
}
/// <summary>
/// Gets the next message from the queue, asynchronously.
/// </summary>
/// <param name="messageOptions">The message options object.</param>
/// <param name="messageSpecificCancellationTokenSource">The message specific cancellation token source.</param>
/// <returns>An <see cref="IQueueMessage" /> instance.</returns>
protected internal virtual async Task <IQueueMessage> GetMessageFromQueue(HandleMessagesSerialOptions messageOptions, CancellationTokenSource messageSpecificCancellationTokenSource)
{
return(await this.GetMessageAsync(messageOptions.MessageLeaseTime, messageSpecificCancellationTokenSource.Token).ConfigureAwait(false));
}