/// <summary>
/// Start the timer.
/// </summary>
private void TriggerTimer()
{
if (this.stop)
{
lock (this.timerStopLock)
{
if (this.stop)
{
this.timerStopped = true;
BrokerTracing.TraceInfo(
SoaHelper.CreateTraceMessage(
"MessageSender.Worker",
"TriggerTimer",
"Worker stops."));
return;
}
else
{
this.timerStopped = false;
}
}
}
try
{
BrokerTracing.TraceInfo(
SoaHelper.CreateTraceMessage(
"MessageSender.Worker",
"TriggerTimer",
"Timer triggered with sleep time {0}.", this.sleepPeriod));
this.timer.Change(this.sleepPeriod, Timeout.Infinite);
if (this.sleepPeriod == 0)
{
this.sleepPeriod = MinSleepTime;
}
else
{
this.sleepPeriod *= 2;
if (this.sleepPeriod > MaxSleepTime)
{
this.sleepPeriod = MaxSleepTime;
}
}
}
catch (NullReferenceException)
{
TraceUtils.TraceWarning(
"MessageSender.Worker",
"TriggerTimer",
"NullReferenceException occurs when timer is being disposed.");
}
catch (Exception e)
{
TraceUtils.TraceError("MessageSender.Worker", "TriggerTimer", "Error occurs, {0}", e);
}
}
/// <summary>
/// Invoke callback.
/// </summary>
/// <param name="asyncResults">a collection of IAsyncResult</param>
private void InvokeCallback(ICollection <IAsyncResult> asyncResults)
{
try
{
asyncResults.AsParallel <IAsyncResult>().ForAll <IAsyncResult>(
(asyncResult) =>
{
try
{
var reliableState = asyncResult.AsyncState as ReliableState;
if (!reliableState.CallbackInvoked())
{
TraceUtils.TraceWarning(
"ReliableQueueClient",
"InvokeCallback",
"Timeout happens, expected trigger time of message {0} is {1}, client {2}",
reliableState.MessageId,
reliableState.TriggerTime.ToLocalTime(),
this.clientId);
reliableState.Timeout = true;
reliableState.Callback(asyncResult);
}
else
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"InvokeCallback",
"Callback of message {0} is already invoked, client {1}",
reliableState.MessageId,
this.clientId);
}
}
catch (Exception e)
{
TraceUtils.TraceError(
"ReliableQueueClient",
"InvokeCallback",
"Invoking callback failed, {0}, client {1}",
e,
this.clientId);
}
});
}
catch (Exception e)
{
TraceUtils.TraceError(
"ReliableQueueClient",
"InvokeCallback",
"Error occurs, {0}, client {1}",
e,
this.clientId);
}
}
/// <summary>
/// Async method for delete message.
/// </summary>
/// <param name="asyncResult">async result</param>
private void EndDeleteMessage(IAsyncResult asyncResult)
{
UniqueId messageId = null;
try
{
Tuple <object, UniqueId> tuple = asyncResult.AsyncState as Tuple <object, UniqueId>;
messageId = tuple.Item2;
// Notice: (azure burst) for perf consideration, on-premise
// broker does not need to delete blob (for large message),
// they will be removed when broker terminates. And broker
// service also conducts cleanup.
if (this.onAzure)
{
if (messageId != null)
{
TraceUtils.TraceVerbose(
"AzureStorageClient",
"EndDeleteMessage",
"Delete message {0} from storage blob.",
messageId);
try
{
this.DeleteBlob(messageId);
}
catch (Exception e)
{
TraceUtils.TraceError("AzureStorageClient", "EndDeleteMessage", "DeleteBlob failed, {0}", e);
}
}
}
this.Queue.EndDeleteMessage(asyncResult);
}
catch (Exception e)
{
string messageIdString = string.Empty;
if (messageId != null)
{
messageIdString = messageId.ToString();
}
TraceUtils.TraceError(
"AzureStorageClient",
"EndDeleteMessage",
"Error occurs when delete message {0}, {1}",
messageIdString,
e);
}
}
/// <summary>
/// Start the timer.
/// </summary>
private void TriggerTimer()
{
if (this.stop)
{
TraceUtils.TraceWarning(
"MessageRetriever.Worker",
"TriggerTimer",
"Worker stops, worker {0}, queue {1}",
this.workerId,
this.queue.Name);
return;
}
try
{
this.timer.Change(this.sleepPeriod, Timeout.Infinite);
if (this.sleepPeriod == 0)
{
this.sleepPeriod = MinSleepTime;
}
else
{
this.sleepPeriod *= 2;
if (this.sleepPeriod > MaxSleepTime)
{
this.sleepPeriod = MaxSleepTime;
}
}
}
catch (NullReferenceException)
{
TraceUtils.TraceWarning(
"MessageRetriever.Worker",
"TriggerTimer",
"NullReferenceException occurs when timer is being disposed, worker {0}, queue {1}",
this.workerId,
this.queue.Name);
}
catch (Exception e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"TriggerTimer",
"Error occurs, worker {0}, queue {1}, {2}",
this.workerId,
this.queue.Name,
e);
}
}
/// <summary>
/// Async method for add message.
/// </summary>
/// <param name="asyncResult">async result</param>
public void EndAddMessage(IAsyncResult asyncResult)
{
var reliableState = asyncResult.AsyncState as ReliableState;
if (reliableState.Timeout)
{
throw new TimeoutException(
string.Format(
"Callback is lost after {0} minutes, client {1}",
CallbackTimeout.TotalMinutes,
this.clientId));
}
if (!reliableState.CallbackInvoked())
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"EndAddMessage",
"Message {0}, client {1}",
reliableState.MessageId,
this.clientId);
try
{
this.cloudQueue.EndAddMessage(asyncResult);
}
catch (Exception e)
{
TraceUtils.TraceError(
"ReliableQueueClient",
"EndAddMessage",
"EndAddMessage failed with excpetion {0}, client {1}",
e,
this.clientId);
}
finally
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"EndAddMessage",
"Try remove the message {0} from cache, client {1}",
reliableState.MessageId,
this.clientId);
IAsyncResult tmp;
this.asyncResultCache.TryRemove(reliableState.MessageId, out tmp);
}
}
}
/// <summary>
/// Dispose the object.
/// </summary>
/// <param name="disposing">disposing flag</param>
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
if (disposing)
{
try
{
// set stop flag
this.stop = true;
if (this.timer != null)
{
try
{
this.timer.Dispose();
}
catch (Exception ex)
{
// ignore the error here
TraceUtils.TraceWarning("MessageSender.Worker", "Dispose", "Exception while disposing timer {0}", ex);
}
this.timer = null;
}
// wait for callback to be called in case there is
// on-the-fly async call
if (!this.waitHandler.Wait(TimeSpan.FromSeconds(5)))
{
TraceUtils.TraceWarning("MessageSender.Worker", "Dispose", "WaitHandler.Wait timeout happens.");
}
try
{
this.waitHandler.Dispose();
}
catch
{
}
this.waitHandler = null;
}
catch (Exception e)
{
TraceUtils.TraceError("MessageSender.Worker", "Dispose", "Error occurs, {0}", e);
}
}
}
private void CheckWorker(object obj)
{
try
{
bool anyResponses = false;
foreach (Tuple <ConcurrentQueue <Message>, AzureStorageClient> responseMessageClient in this.responseMessageClients.Values)
{
if (responseMessageClient.Item1.Count > 0)
{
anyResponses = true;
break;
}
}
if (anyResponses)
{
if (!this.IsRunning)
{
this.Start();
this.IsRunning = true;
}
}
else
{
if (this.IsRunning)
{
this.Pause();
this.IsRunning = false;
}
}
if (this.IsRunning)
{
this.timer.Change(SleepTimeWhenRunning, Timeout.Infinite);
}
else
{
this.timer.Change(SleepTimeWhenStopped, Timeout.Infinite);
}
}
catch (Exception e)
{
TraceUtils.TraceError("MessageSender.Worker", "CheckWorker", "Error occurs, {0}", e);
}
}
/// <summary>
/// Dispose current object.
/// </summary>
/// <param name="disposing">disposing flag</param>
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
if (disposing)
{
if (this.timer != null)
{
try
{
this.timer.Dispose();
}
catch (Exception e)
{
TraceUtils.TraceError(
"ReliableQueueClient",
"Dispose",
"Dispose timer failed, {0}, client {1}",
e,
this.clientId);
}
this.timer = null;
}
// Trigger TimeoutException on each item in asyncResultQueue.
// Otherwise if callback lost, we don't have a way to recover
// from that.
int count = this.asyncResultCache.Count;
if (count > 0)
{
TraceUtils.TraceWarning(
"ReliableQueueClient",
"Dispose",
"Trigger TimeoutException on each item in asyncResultQueue, count {0}, client {1}",
count,
this.clientId);
this.InvokeCallback(this.asyncResultCache.Values);
}
}
}
/// <summary>
/// Invoke invalidQueueHandler if failed to access the queue.
/// </summary>
/// <remarks>
/// Notice: Only handle the case that queue is not found now. May
/// add more error handlings for specific queue issues if necessary.
/// </remarks>
/// <param name="e">exception happens when access the queue</param>
private void HandleInvalidQueue(StorageException e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"HandleInvalidQueue",
"StorageException, worker {0}, queue {1}, error code {2}, {3}",
this.workerId,
this.queue.Name,
BurstUtility.GetStorageErrorCode(e),
e);
if (BurstUtility.IsQueueNotFound(e))
{
// Invoke invalidQueueHandler if the exception indicates
// that the queue is not found.
if (this.invalidQueueHandler != null)
{
this.invalidQueueHandler(e);
}
}
}
/// <summary>
/// Invoke the invalidQueueHandler.
/// </summary>
/// <param name="e">exception occurred when access the queue</param>
private void HandleInvalidQueue(StorageException e)
{
if (this.invalidQueueHandler != null)
{
if (Interlocked.CompareExchange(ref this.invalidQueueHandlerInvoked, 1, 0) == 0)
{
try
{
// invoke the handler in threadpool thread rather than
// in IO completion thread
ThreadPool.QueueUserWorkItem((s) => { this.invalidQueueHandler(e); });
}
catch (Exception ex)
{
TraceUtils.TraceError(
"MessageRetriever",
"HandleInvalidQueue",
"Error occurs, queue {0}, {1}",
this.queueName,
ex);
}
}
}
}
/// <summary>
/// Callback of the BeginGetMessages method.
/// </summary>
/// <param name="ar">async result</param>
private void GetMessagesCallback(IAsyncResult ar)
{
Guid callId = (Guid)ar.AsyncState;
try
{
IEnumerable <CloudQueueMessage> messages = null;
try
{
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"GetMessagesCallback",
"Call EndGetMessages, worker {0}, call Id {1}, queue {2}",
this.workerId,
callId,
this.queue.Name);
messages = this.queue.EndGetMessages(ar);
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"GetMessagesCallback",
"EndGetMessages returns, worker {0}, call Id {1}, queue {2}",
this.workerId,
callId,
this.queue.Name);
}
catch (StorageException e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"GetMessagesCallback",
"EndGetMessages failed, worker {0}, call Id {1}, queue {2}, error code {3}, {4}",
this.workerId,
callId,
this.queue.Name,
BurstUtility.GetStorageErrorCode(e),
e);
this.HandleInvalidQueue(e);
}
catch (Exception e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"GetMessagesCallback",
"EndGetMessages failed, worker {0}, call Id {1}, queue {2}, {3}",
this.workerId,
callId,
this.queue.Name,
e);
}
finally
{
this.waitHandler.Set();
}
int count = 0;
if (messages != null)
{
count = messages.Count <CloudQueueMessage>();
}
if (count > 0)
{
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"GetMessagesCallback",
"Get {0} messages from the queue, worker {1}, call Id {2}, queue {3}",
count,
this.workerId,
callId,
this.queue.Name);
this.sleepPeriod = 0;
// Make sure messageHandler is a fast operation, call
// it before getting messages next time, in case
// current thread doesn't get chance on time to call
// messageHandler if BeginGetMessages's callback is
// invoked on current thread again.
if (this.messageHandler != null)
{
try
{
this.messageHandler(messages);
}
catch (Exception e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"GetMessagesCallback",
"Message handler throws exception, worker {0}, call Id {1}, queue {2}, {3}",
this.workerId,
callId,
this.queue.Name,
e);
}
}
this.InternalBeginGetMessages(null);
}
else
{
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"GetMessagesCallback",
"Get 0 message from the queue, worker {0}, call Id {1}, queue {2}",
this.workerId,
callId,
this.queue.Name);
this.TriggerTimer();
}
}
catch (Exception e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"GetMessagesCallback",
"Error occurs, worker {0}, call Id {1}, queue {2}, {3}",
this.workerId,
callId,
this.queue.Name,
e);
this.TriggerTimer();
}
}
/// <summary>
/// Call cloud queue's async method to get messages.
/// </summary>
/// <param name="state">state object</param>
private void InternalBeginGetMessages(object state)
{
if (this.stop)
{
// if current worker already stops, just return
return;
}
// this Id is only used in trace log to track latency for each
// BeginGetMessages method call
Guid callId = Guid.NewGuid();
try
{
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"InternalGetMessages",
"Call BeginGetMessages, worker {0}, call Id {1}, queue {2}",
this.workerId,
callId,
this.queue.Name);
this.waitHandler.Reset();
this.queue.BeginGetMessages(Constant.GetQueueMessageBatchSize, this.visibleTimeout, null, null, this.GetMessagesCallback, callId);
TraceUtils.TraceVerbose(
"MessageRetriever.Worker",
"InternalGetMessages",
"BeginGetMessages returns, worker {0}, call Id {1}, queue {2}",
this.workerId,
callId,
this.queue.Name);
}
catch (StorageException e)
{
this.waitHandler.Set();
TraceUtils.TraceError(
"MessageRetriever.Worker",
"InternalGetMessages",
"BeginGetMessages failed, worker {0}, call Id {1}, queue {2}, error code {3}, {4}",
this.workerId,
callId,
this.queue.Name,
BurstUtility.GetStorageErrorCode(e),
e);
this.HandleInvalidQueue(e);
this.TriggerTimer();
}
catch (Exception e)
{
this.waitHandler.Set();
TraceUtils.TraceError(
"MessageRetriever.Worker",
"InternalGetMessages",
"Error occurs, worker {0}, call Id {1}, queue {2}, {3}",
this.workerId,
callId,
this.queue.Name,
e);
this.TriggerTimer();
}
}
/// <summary>
/// Dispose the object.
/// </summary>
/// <param name="disposing">disposing flag</param>
protected override void Dispose(bool disposing)
{
base.Dispose(disposing);
if (disposing)
{
try
{
// set stop flag
this.stop = true;
if (this.timer != null)
{
try
{
this.timer.Dispose();
}
catch (Exception ex)
{
TraceUtils.TraceWarning(
"MessageRetriever.Worker",
"Dispose",
"Disposing timer exception {0}",
ex);
}
this.timer = null;
}
// wait for callback to be called in case there is
// on-the-fly async call
if (!this.waitHandler.Wait(TimeSpan.FromSeconds(5)))
{
TraceUtils.TraceWarning(
"MessageRetriever.Worker",
"Dispose",
"WaitHandler.Wait timeout happens, worker {0}, queue {1}",
this.workerId,
this.queue.Name);
}
try
{
this.waitHandler.Dispose();
}
catch (Exception ex)
{
TraceUtils.TraceWarning(
"MessageRetriever.Worker",
"Dispose",
"Disposing waitHandler exception {0}",
ex);
}
this.waitHandler = null;
}
catch (Exception e)
{
TraceUtils.TraceError(
"MessageRetriever.Worker",
"Dispose",
"Error occurs, worker {0}, queue {1}, {2}",
this.workerId,
this.queue.Name,
e);
}
}
}
/// <summary>
/// Callback method of the timer.
/// </summary>
/// <param name="state">state object</param>
private void TimerCallback(object state)
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"Enter timer callback, client {0}",
this.clientId);
List <IAsyncResult> lostCallbacks = new List <IAsyncResult>();
List <UniqueId> messageIds = new List <UniqueId>();
try
{
IAsyncResult result;
DateTime utcNow = DateTime.UtcNow;
var keys = this.asyncResultCache.Keys;
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"There are {0} messages in the local cache, client {1}",
keys.Count,
this.clientId);
foreach (UniqueId messageId in keys)
{
if (this.asyncResultCache.TryGetValue(messageId, out result))
{
var reliableState = result.AsyncState as ReliableState;
if (reliableState.TriggerTime <= utcNow)
{
lostCallbacks.Add(result);
messageIds.Add(messageId);
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"Expected trigger time of message {0} is {1}, get it, client {2}",
messageId,
reliableState.TriggerTime.ToLocalTime(),
this.clientId);
}
else
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"Expected trigger time of message {0} is {1}, ignore it, client {2}",
messageId,
reliableState.TriggerTime.ToLocalTime(),
this.clientId);
}
}
}
// invoke the lost callbacks before deleting them from cache
if (lostCallbacks.Count > 0)
{
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"Invoke callbacks, count {0}, client {1}",
lostCallbacks.Count,
this.clientId);
this.InvokeCallback(lostCallbacks);
}
// delete callbacks from cache
if (messageIds.Count > 0)
{
foreach (UniqueId messageId in messageIds)
{
IAsyncResult tmp;
this.asyncResultCache.TryRemove(messageId, out tmp);
TraceUtils.TraceVerbose(
"ReliableQueueClient",
"TimerCallback",
"Remove message {0} from cache, client {1}",
messageId,
this.clientId);
}
}
}
catch (Exception e)
{
TraceUtils.TraceError(
"ReliableQueueClient",
"TimerCallback",
"Error occurs, {0}, client {1}",
e,
this.clientId);
}
}