/// <summary>
/// Invoke callback.
/// </summary>
/// <param name="asyncResults">a collection of IAsyncResult</param>
private void InvokeCallback(ICollection <IAsyncResult> asyncResults)
{
try
{
asyncResults.AsParallel <IAsyncResult>().ForAll <IAsyncResult>(
(asyncResult) =>
{
try
{
var reliableState = asyncResult.AsyncState as ReliableState;
if (!reliableState.CallbackInvoked())
{
TraceUtils.TraceWarning(
"ReliableQueueClient",
"InvokeCallback",
"Timeout happens, expected trigger time of message {0} is {1}, client {2}",
reliableState.MessageId,
reliableState.TriggerTime.ToLocalTime(),
this.clientId);
reliableState.Timeout = true;
reliableState.Callback(asyncResult);
}
else
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"InvokeCallback",
"Callback of message {0} is already invoked, client {1}",
reliableState.MessageId,
this.clientId);
}
}
catch (Exception e)
{
TraceUtils.TraceError(
"ReliableQueueClient",
"InvokeCallback",
"Invoking callback failed, {0}, client {1}",
e,
this.clientId);
}
});
}
catch (Exception e)
{
TraceUtils.TraceError(
"ReliableQueueClient",
"InvokeCallback",
"Error occurs, {0}, client {1}",
e,
this.clientId);
}
}
/// <summary>
/// Async method for delete message.
/// </summary>
/// <param name="asyncResult">async result</param>
private void EndDeleteMessage(IAsyncResult asyncResult)
{
UniqueId messageId = null;
try
{
Tuple <object, UniqueId> tuple = asyncResult.AsyncState as Tuple <object, UniqueId>;
messageId = tuple.Item2;
// Notice: (azure burst) for perf consideration, on-premise
// broker does not need to delete blob (for large message),
// they will be removed when broker terminates. And broker
// service also conducts cleanup.
if (this.onAzure)
{
if (messageId != null)
{
TraceUtils.TraceVerbose(
"AzureStorageClient",
"EndDeleteMessage",
"Delete message {0} from storage blob.",
messageId);
try
{
this.DeleteBlob(messageId);
}
catch (Exception e)
{
TraceUtils.TraceError("AzureStorageClient", "EndDeleteMessage", "DeleteBlob failed, {0}", e);
}
}
}
this.Queue.EndDeleteMessage(asyncResult);
}
catch (Exception e)
{
string messageIdString = string.Empty;
if (messageId != null)
{
messageIdString = messageId.ToString();
}
TraceUtils.TraceError(
"AzureStorageClient",
"EndDeleteMessage",
"Error occurs when delete message {0}, {1}",
messageIdString,
e);
}
}
/// <summary>
/// Async method for add message.
/// </summary>
/// <param name="asyncResult">async result</param>
public void EndAddMessage(IAsyncResult asyncResult)
{
var reliableState = asyncResult.AsyncState as ReliableState;
if (reliableState.Timeout)
{
throw new TimeoutException(
string.Format(
"Callback is lost after {0} minutes, client {1}",
CallbackTimeout.TotalMinutes,
this.clientId));
}
if (!reliableState.CallbackInvoked())
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"EndAddMessage",
"Message {0}, client {1}",
reliableState.MessageId,
this.clientId);
try
{
this.cloudQueue.EndAddMessage(asyncResult);
}
catch (Exception e)
{
TraceUtils.TraceError(
"ReliableQueueClient",
"EndAddMessage",
"EndAddMessage failed with excpetion {0}, client {1}",
e,
this.clientId);
}
finally
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"EndAddMessage",
"Try remove the message {0} from cache, client {1}",
reliableState.MessageId,
this.clientId);
IAsyncResult tmp;
this.asyncResultCache.TryRemove(reliableState.MessageId, out tmp);
}
}
}
/// <summary>
/// Async method for add message. If the message size hits limit, store
/// it in blob and add a referral message in queue.
/// </summary>
/// <param name="messageData">raw data of message</param>
/// <param name="messageId">wcf message Id</param>
/// <param name="callback">callback method</param>
/// <param name="state">state object</param>
/// <returns>async state</returns>
public IAsyncResult BeginAddMessage(byte[] messageData, UniqueId messageId, AsyncCallback callback, ReliableQueueClient.ReliableState state)
{
CloudQueueMessage message = new CloudQueueMessage(messageData);
try
{
if (messageData.Length <= MessageSizeBoundary)
{
message = new CloudQueueMessage(messageData);
return(this.ReliableQueue.BeginAddMessage(message, callback, state));
}
}
catch (ArgumentException e)
{
// according to the test, when payload is <= 48KB, it can fit
// in a queue message. otherwise, ArgumentException occurs. but
// there is no doc about this. so catch ArgumentException here,
// and store message to blob below.
TraceUtils.TraceWarning("AzureStorageClient", "BeginAddMessage", "BeginAddMessage failed, {0}", e);
}
catch (Exception ex)
{
TraceUtils.TraceWarning("AzureStorageClient", "BeginAddMessage", "BeginAddMessage failed, {0}", ex);
throw;
}
TraceUtils.TraceVerbose("AzureStorageClient", "BeginAddMessage", "Upload message {0} to storage blob.", messageId);
this.UploadBlob(messageData, messageId);
// Store a referral message in Azure queue with the same message Id
// as the original message. It redirects proxy to get real message
// from the blob.
Message referralMessage = Message.CreateMessage(MessageVersion.Default, string.Empty);
referralMessage.Headers.MessageId = messageId;
referralMessage.Headers.Add(
MessageHeader.CreateHeader(Constant.MessageHeaderBlob, Constant.HpcHeaderNS, string.Empty));
message = new CloudQueueMessage(AzureQueueItem.Serialize(referralMessage));
return(this.ReliableQueue.BeginAddMessage(message, callback, state));
}
/// <summary>
/// Async method for add message to queue.
/// </summary>
/// <param name="message">queue message</param>
/// <param name="callback">callback method</param>
/// <param name="state">state object</param>
/// <returns>async result</returns>
public IAsyncResult BeginAddMessage(CloudQueueMessage message, AsyncCallback callback, ReliableState state)
{
state.Callback = callback;
state.TriggerTime = DateTime.UtcNow + CallbackTimeout;
IAsyncResult result = this.cloudQueue.BeginAddMessage(message, callback, state);
this.asyncResultCache.AddOrUpdate(state.MessageId, result, (key, value) => result);
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"BeginAddMessage",
"Add message {0} to the local cache, client {1}",
state.MessageId,
this.clientId);
return(result);
}
/// <summary>
/// Dispose current object.
/// </summary>
/// <param name="disposing">disposing flag</param>
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
if (disposing)
{
if (this.ReliableQueue != null)
{
try
{
this.ReliableQueue.Close();
}
catch (Exception e)
{
TraceUtils.TraceVerbose("AzureStorageClient", "Dispose", "Closing ReliableQueue failed, {0}", e);
}
this.ReliableQueue = null;
}
}
}
/// <summary>
/// Callback of the BeginGetMessages method.
/// </summary>
/// <param name="ar">async result</param>
private void GetMessagesCallback(IAsyncResult ar)
{
Guid callId = (Guid)ar.AsyncState;
try
{
IEnumerable <CloudQueueMessage> messages = null;
try
{
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"GetMessagesCallback",
"Call EndGetMessages, worker {0}, call Id {1}, queue {2}",
this.workerId,
callId,
this.queue.Name);
messages = this.queue.EndGetMessages(ar);
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"GetMessagesCallback",
"EndGetMessages returns, worker {0}, call Id {1}, queue {2}",
this.workerId,
callId,
this.queue.Name);
}
catch (StorageException e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"GetMessagesCallback",
"EndGetMessages failed, worker {0}, call Id {1}, queue {2}, error code {3}, {4}",
this.workerId,
callId,
this.queue.Name,
BurstUtility.GetStorageErrorCode(e),
e);
this.HandleInvalidQueue(e);
}
catch (Exception e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"GetMessagesCallback",
"EndGetMessages failed, worker {0}, call Id {1}, queue {2}, {3}",
this.workerId,
callId,
this.queue.Name,
e);
}
finally
{
this.waitHandler.Set();
}
int count = 0;
if (messages != null)
{
count = messages.Count <CloudQueueMessage>();
}
if (count > 0)
{
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"GetMessagesCallback",
"Get {0} messages from the queue, worker {1}, call Id {2}, queue {3}",
count,
this.workerId,
callId,
this.queue.Name);
this.sleepPeriod = 0;
// Make sure messageHandler is a fast operation, call
// it before getting messages next time, in case
// current thread doesn't get chance on time to call
// messageHandler if BeginGetMessages's callback is
// invoked on current thread again.
if (this.messageHandler != null)
{
try
{
this.messageHandler(messages);
}
catch (Exception e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"GetMessagesCallback",
"Message handler throws exception, worker {0}, call Id {1}, queue {2}, {3}",
this.workerId,
callId,
this.queue.Name,
e);
}
}
this.InternalBeginGetMessages(null);
}
else
{
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"GetMessagesCallback",
"Get 0 message from the queue, worker {0}, call Id {1}, queue {2}",
this.workerId,
callId,
this.queue.Name);
this.TriggerTimer();
}
}
catch (Exception e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"GetMessagesCallback",
"Error occurs, worker {0}, call Id {1}, queue {2}, {3}",
this.workerId,
callId,
this.queue.Name,
e);
this.TriggerTimer();
}
}
/// <summary>
/// Call cloud queue's async method to get messages.
/// </summary>
/// <param name="state">state object</param>
private void InternalBeginGetMessages(object state)
{
if (this.stop)
{
// if current worker already stops, just return
return;
}
// this Id is only used in trace log to track latency for each
// BeginGetMessages method call
Guid callId = Guid.NewGuid();
try
{
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"InternalGetMessages",
"Call BeginGetMessages, worker {0}, call Id {1}, queue {2}",
this.workerId,
callId,
this.queue.Name);
this.waitHandler.Reset();
this.queue.BeginGetMessages(Constant.GetQueueMessageBatchSize, this.visibleTimeout, null, null, this.GetMessagesCallback, callId);
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"InternalGetMessages",
"BeginGetMessages returns, worker {0}, call Id {1}, queue {2}",
this.workerId,
callId,
this.queue.Name);
}
catch (StorageException e)
{
this.waitHandler.Set();
TraceUtils.TraceError(
"MessageRetriever.Worker",
"InternalGetMessages",
"BeginGetMessages failed, worker {0}, call Id {1}, queue {2}, error code {3}, {4}",
this.workerId,
callId,
this.queue.Name,
BurstUtility.GetStorageErrorCode(e),
e);
this.HandleInvalidQueue(e);
this.TriggerTimer();
}
catch (Exception e)
{
this.waitHandler.Set();
TraceUtils.TraceError(
"MessageRetriever.Worker",
"InternalGetMessages",
"Error occurs, worker {0}, call Id {1}, queue {2}, {3}",
this.workerId,
callId,
this.queue.Name,
e);
this.TriggerTimer();
}
}
/// <summary>
/// Callback method of the timer.
/// </summary>
/// <param name="state">state object</param>
private void TimerCallback(object state)
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"Enter timer callback, client {0}",
this.clientId);
List <IAsyncResult> lostCallbacks = new List <IAsyncResult>();
List <UniqueId> messageIds = new List <UniqueId>();
try
{
IAsyncResult result;
DateTime utcNow = DateTime.UtcNow;
var keys = this.asyncResultCache.Keys;
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"There are {0} messages in the local cache, client {1}",
keys.Count,
this.clientId);
foreach (UniqueId messageId in keys)
{
if (this.asyncResultCache.TryGetValue(messageId, out result))
{
var reliableState = result.AsyncState as ReliableState;
if (reliableState.TriggerTime <= utcNow)
{
lostCallbacks.Add(result);
messageIds.Add(messageId);
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"Expected trigger time of message {0} is {1}, get it, client {2}",
messageId,
reliableState.TriggerTime.ToLocalTime(),
this.clientId);
}
else
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"Expected trigger time of message {0} is {1}, ignore it, client {2}",
messageId,
reliableState.TriggerTime.ToLocalTime(),
this.clientId);
}
}
}
// invoke the lost callbacks before deleting them from cache
if (lostCallbacks.Count > 0)
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"Invoke callbacks, count {0}, client {1}",
lostCallbacks.Count,
this.clientId);
this.InvokeCallback(lostCallbacks);
}
// delete callbacks from cache
if (messageIds.Count > 0)
{
foreach (UniqueId messageId in messageIds)
{
IAsyncResult tmp;
this.asyncResultCache.TryRemove(messageId, out tmp);
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"Remove message {0} from cache, client {1}",
messageId,
this.clientId);
}
}
}
catch (Exception e)
{
TraceUtils.TraceError(
"ReliableQueueClient",
"TimerCallback",
"Error occurs, {0}, client {1}",
e,
this.clientId);
}
}