/// <summary>
/// Get a response from the storage
/// </summary>
/// <param name="callback">the response callback, the async result should be the BrokerQueueItem</param>
/// <param name="callbackState">the state object for the callback</param>
public void GetResponseAsync(PersistCallback callback, object callbackState)
{
if (this.isClosedField)
{
BrokerTracing.TraceWarning("[MemoryPersist] .GetResponseAsync: the queue is closed.");
return;
}
BrokerQueueItem responseItem = null;
try
{
lock (this.lockResponseQueueField)
{
responseItem = this.responseQueueField.Dequeue();
}
}
catch (InvalidOperationException)
{
BrokerTracing.TraceError("[MemoryPersist] .GetResponseAsync: no response available. sessionId = {0}, clientId = {1}", this.sessionIdField, this.clientIdField);
}
if (callback == null)
{
return;
}
ThreadPool.QueueUserWorkItem(this.GetOperationComplete, new GetCallbackContext(callback, callbackState, responseItem));
}
/// <summary>
/// Add QueueAsyncResult to the dictionary.
/// </summary>
/// <param name="result">async result</param>
/// <param name="requestQueueName">request queue name</param>
/// <param name="responseQueueName">response queue name</param>
public void AddQueueAsyncResult(QueueAsyncResult result, string requestQueueName, string responseQueueName)
{
ResponseStorageException e;
this.responseQueueNotFound.TryGetValue(responseQueueName, out e);
if (e == null)
{
BrokerTracing.TraceVerbose(
"[AzureQueueManager].AddQueueAsyncResult: Add QueueAsyncResult {0} to the dictionary.",
result.MessageId);
this.callbacks.AddOrUpdate(result.MessageId, result, (key, value) => result);
this.requestsMappingToRequestQueue.AddOrUpdate(result.MessageId, requestQueueName, (key, value) => requestQueueName);
this.requestsMappingToResponseQueue.AddOrUpdate(result.MessageId, responseQueueName, (key, value) => responseQueueName);
}
else
{
BrokerTracing.TraceError(
"[AzureQueueManager].AddQueueAsyncResult: Response queue is not found, {0}",
responseQueueName);
throw e;
}
}
public static async Task DeleteTableAsync(string connectString, string sessionId, string responseTableName)
{
var info = new QueueInfo(sessionId, responseTableName);
info.ETag = "*"; // no etag cannot delete.
try
{
var table = GetTableClient(connectString).GetTableReference(queueTableName);
var deleteOperation = TableOperation.Delete(info);
await table.ExecuteAsync(deleteOperation);
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureStorageTool] .DeleteTableAsync: Table={0} delete from storage table failed, the exception, {1}",
responseTableName,
e);
throw;
}
try
{
var responseTable = GetTableClient(connectString).GetTableReference(responseTableName);
await responseTable.DeleteIfExistsAsync();
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureStorageTool] .DeleteTableAsync: Table={0} delete failed, the exception, {1}",
responseTableName,
e);
throw;
}
}
/// <summary>
/// Async method of ProcessMessage.
/// </summary>
/// <param name="ar">async result</param>
/// <returns>response message</returns>
public Message EndProcessMessage(IAsyncResult ar)
{
QueueAsyncResult asyncResult = ar as QueueAsyncResult;
Debug.Assert(asyncResult != null, "ar must be a QueueAsyncResult.");
using (asyncResult)
{
UniqueId messageId = asyncResult.MessageId;
try
{
this.manager.RemoveQueueAsyncResult(messageId);
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureServiceClient].EndProcessMessage: Failed to remove QueueAsyncResult {0}, {1}",
messageId,
e);
}
if (asyncResult.Exception != null)
{
throw asyncResult.Exception;
}
else
{
return(asyncResult.ResponseMessage);
}
}
}
/// <summary>
/// Add a dispatcher to blocked dispatcher list
/// </summary>
/// <param name="dispatcherInfo">dispatcher info</param>
private void AddBlockedDispatcher(DispatcherInfo dispatcherInfo)
{
try
{
dispatcherInfo.BlockTime = DateTime.Now;
this.blockedDispatcherDic.Add(dispatcherInfo.UniqueId, dispatcherInfo);
// if a dispatcher has never been blocked, take it as "young" blocked disaptcher.
if (dispatcherInfo.BlockRetryCount <= 0)
{
BrokerTracing.TraceVerbose("[DispatcherManager] Increment youngBlockedDispatcherCount, task id={0}, BlockRetryCount={1}", dispatcherInfo.UniqueId, dispatcherInfo.BlockRetryCount);
this.youngBlockedDispatcherCount++;
}
this.blockedDispatcherQueue.Enqueue(dispatcherInfo);
if (this.blockedDispatcherQueue.Count == 1)
{
BrokerTracing.TraceVerbose("[DispatcherManager] Block dispatcher: change unblock timer, task id = {0}", dispatcherInfo.UniqueId);
this.unblockTimer.Change(this.blockTimeSpan, TimeSpan.FromMilliseconds(-1));
}
BrokerTracing.TraceInfo("[DispatcherManager] Add dispatcher {0} into the blocked dispatcher list.", dispatcherInfo.UniqueId);
}
catch (ArgumentException)
{
BrokerTracing.TraceError("[DispatcherManager] Dispatcher {0} already exist in the blocked dispatcher list.", dispatcherInfo.UniqueId);
}
}
/// <summary>
/// Dispose the object
/// </summary>
void IDisposable.Dispose()
{
int count = Interlocked.Decrement(ref this.refCount);
if (count < 0)
{
BrokerTracing.TraceError("[ReferenceObject] Ref count is {0} in dispose method of {1} class", count, this.GetType());
return;
}
else if (count == 0)
{
#if DEBUG
// In debug build, decrease the total count when disposing
Interlocked.Decrement(ref totalCount);
#endif
this.Dispose(true);
GC.SuppressFinalize(this);
}
else
{
// If the real dispose method is not done by the call, wait until the real procedure is done
// This wait handle makes sure that the real dispose procedure is done after the Dispose method is called, which guaranteed the dispose order
this.disposeWaitHandle.WaitOne();
}
// Dispose the wait handle at last
this.disposeWaitHandle.Close();
this.disposeWaitHandle = null;
}
/// <summary>
/// Remove QueueAsyncResult from the dictionary.
/// </summary>
/// <param name="messageId">message Id</param>
public void RemoveQueueAsyncResult(UniqueId messageId)
{
string value;
if (!this.requestsMappingToRequestQueue.TryRemove(messageId, out value))
{
BrokerTracing.TraceError(
"[AzureQueueManager].CompleteCallback: Failed to remove message Id {0} from requestsMappingToRequestQueue.",
messageId);
}
if (!this.requestsMappingToResponseQueue.TryRemove(messageId, out value))
{
BrokerTracing.TraceError(
"[AzureQueueManager].CompleteCallback: Failed to remove message Id {0} from requestsMappingToResponseQueue.",
messageId);
}
QueueAsyncResult result;
if (!this.callbacks.TryRemove(messageId, out result))
{
BrokerTracing.TraceError(
"[AzureQueueManager].CompleteCallback: Failed to remove QueueAsyncResult {0} from callbacks.",
messageId);
}
else
{
BrokerTracing.TraceVerbose(
"[AzureQueueManager].CompleteCallback: Remove QueueAsyncResult {0} from callbacks.",
messageId);
}
}
/// <summary>
/// Gets the broker frontend
/// </summary>
/// <param name="callbackInstance">indicating response service callback</param>
/// <returns>returns the broker frontend instance</returns>
public IBrokerFrontend GetFrontendForInprocessBroker(IResponseServiceCallback callbackInstance)
{
BrokerTracing.TraceInfo("[BrokerEntry] GetFrontendForInprocessBroker...");
try
{
if (this.closeFlag != 0)
{
if (this.cleanData)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_BrokerUnavailable, SR.BrokerIsUnavailable);
}
else
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_BrokerSuspending, SR.BrokerSuspending);
}
}
this.sharedData.WaitForInitializationComplete();
BrokerTracing.TraceInfo("[BrokerEntry] GetFrontendForInprocessBroker successfully.");
return(new BrokerController(this.clientManager, callbackInstance, this.observer));
}
catch (Exception e)
{
BrokerTracing.TraceError("[BrokerEntry] GetFrontendForInprocessBroker failed: {0}", e);
throw;
}
}
/// <summary>
/// Process a collection of queue messages.
/// </summary>
/// <param name="messages">collection of the queue messages</param>
private void ProcessMessages(IEnumerable <CloudQueueMessage> messages)
{
BrokerTracing.TraceInfo(SoaHelper.CreateTraceMessage("Proxy", "ProcessMessages", string.Format("Process {0} messages.", messages.Count <CloudQueueMessage>())));
messages.AsParallel <CloudQueueMessage>()
.ForAll <CloudQueueMessage>(
(requestQueueMessage) =>
{
Message request = null;
try
{
BrokerTracing.TraceInfo(string.Format("SOA broker proxy perf1 - {0}", DateTime.UtcNow.TimeOfDay.TotalSeconds));
request = this.requestStorageClients.First().GetWcfMessageFromQueueMessage(requestQueueMessage);
this.requestMessageQueue.Add(request);
// this.requestMessageQueue.Enqueue(request);
UniqueId messageId = SoaHelper.GetMessageId(request);
BrokerTracing.TraceInfo(SoaHelper.CreateTraceMessage("Proxy", "Request received inqueue", string.Empty, messageId, "Request message in queue"));
}
catch (Exception e)
{
BrokerTracing.TraceError(SoaHelper.CreateTraceMessage("Proxy", "ProcessMessages", string.Format("Error occurs {0}", e)));
}
});
this.semaphoreForWorker.Release();
}
/// <summary>
/// Informs that a client is attaching to this broker
/// </summary>
public void Attach()
{
BrokerTracing.TraceInfo("[BrokerEntry] Client attached.");
try
{
// Bug 8379: If closing is under going, returns broker suspending exception and let broker manager try again latter.
if (this.closeFlag != 0)
{
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_BrokerSuspending, SR.BrokerSuspending);
}
this.stateManager.Attach();
this.monitor.Attach();
BrokerTracing.TraceInfo("[BrokerEntry] Client attached successfully.");
}
catch (NullReferenceException)
{
// Bug 8379: NullReferenceException caught because closing procedure is on going, returns broker suspending exception instead so that broker manager could properly handle
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_BrokerSuspending, SR.BrokerSuspending);
}
catch (Exception e)
{
BrokerTracing.TraceError("[BrokerEntry] Client attached failed: {0}", e);
throw;
}
}
/// <summary>
/// Get the node mapping for the Azure nodes.
/// </summary>
/// <param name="state">object used by the callback method.</param>
private void GetNodeMappingWorker(object state)
{
try
{
EndpointAddress endpoint = new EndpointAddress(NodeMappingCacheEpr);
using (ChannelFactory <INodeMappingCache> channelFactory = new ChannelFactory <INodeMappingCache>(BindingHelper.HardCodedNamedPipeBinding, endpoint))
{
INodeMappingCache cache = null;
try
{
cache = channelFactory.CreateChannel();
this.dictionary = cache.EndGetNodeMapping(cache.BeginGetNodeMapping(true, null, null));
}
finally
{
if (cache != null)
{
((IClientChannel)cache).Close();
}
}
}
}
catch (Exception e)
{
BrokerTracing.TraceError("[NodeMappingData] .GetNodeMapping: Failed to get node mapping. {0}", e);
}
finally
{
// If error occurs, this.dictionary is null. Set following waithandle avoid blocking the waiter.
this.complete.Set();
}
}
/// <summary>
/// Create storage blob container.
/// </summary>
/// <param name="container">blob container</param>
public static void CreateContainerWithRetry(CloudBlobContainer container)
{
RetryHelper <object> .InvokeOperation(
() =>
{
if (container.CreateIfNotExists())
{
BrokerTracing.TraceInfo("[AzureQueueManager].CreateContainerWithRetry: Create the container {0}", container.Name);
}
return(null);
},
(e, count) =>
{
BrokerTracing.TraceError("Failed to create the container {0}: {1}. Retry Count = {2}", container.Name, e, count);
StorageException se = e as StorageException;
if (se != null)
{
string errorCode = BurstUtility.GetStorageErrorCode(se);
// According to test, the error code is ResourceAlreadyExists.
// There is no doc about this, add ContainerAlreadyExists here.
// TODO: Azure storage SDK 2.0
if ( // errorCode == StorageErrorCodeStrings.ResourceAlreadyExists ||
errorCode == StorageErrorCodeStrings.ContainerAlreadyExists)
{
Thread.Sleep(TimeSpan.FromMinutes(1));
}
}
});
}
/// <summary>
/// Create storage queue
/// </summary>
/// <param name="queue">storage queue</param>
/// <remarks>
/// CreateIfNotExist method throws StorageClientException when queue is
/// being deleted, so sleep for a while before retry.
/// </remarks>
public static void CreateQueueWithRetry(CloudQueue queue)
{
RetryHelper <object> .InvokeOperation(
() =>
{
if (queue.CreateIfNotExists())
{
BrokerTracing.TraceInfo("[AzureQueueManager].CreateQueueWithRetry: Create the queue {0}", queue.Name);
}
return(null);
},
(e, count) =>
{
BrokerTracing.TraceError("Failed to create the queue {0}: {1}. Retry Count = {2}", queue.Name, e, count);
StorageException se = e as StorageException;
if (se != null)
{
if (BurstUtility.GetStorageErrorCode(se) == QueueErrorCodeStrings.QueueAlreadyExists)
{
Thread.Sleep(TimeSpan.FromMinutes(1));
}
}
});
}
/// <summary>
/// Initializes a new instance of the BrokerClientManager class
/// </summary>
/// <param name="clientList">indicating the client info list</param>
/// <param name="queueFactory">indicating the queue factory</param>
/// <param name="observer">indicating the observer</param>
/// <param name="stateManager">indicating the state manager</param>
/// <param name="monitor">indicating the monitor</param>
/// <param name="sharedData">indicating the shared data</param>
public BrokerClientManager(ClientInfo[] clientList, BrokerQueueFactory queueFactory, BrokerObserver observer, BrokerStateManager stateManager, ServiceJobMonitorBase monitor, SharedData sharedData)
{
this.clientDic = new Dictionary <string, BrokerClient>(StringComparer.OrdinalIgnoreCase);
this.queueFactory = queueFactory;
this.observer = observer;
this.stateManager = stateManager;
this.monitor = monitor;
this.sharedData = sharedData;
foreach (ClientInfo client in clientList)
{
try
{
// Bug 5193: Only raise client that has requests to process.
if (client.TotalRequestsCount != client.ProcessedRequestsCount)
{
this.AddNewClient(client.ClientId, client.UserName);
}
}
catch (Exception e)
{
// Create client may fail because of broker queue failure, ignore the client in this situation and trys other client instead.
BrokerTracing.TraceError("[BrokerClientManager] Failed to create client {0}, Exception = {1}", client.ClientId, e);
}
}
this.CheckIfAllRequestDone();
this.CheckIfEOMCalled();
}
public static async Task RestoreRequest(
CloudQueue requestQueue,
CloudQueue pendingQueue,
CloudTable responseTable,
CloudBlobContainer container)
{
try
{
while (true)
{
var message = await pendingQueue.PeekMessageAsync();
if (message == null)
{
break;
}
var messageId =
((BrokerQueueItem)formatter.Deserialize(await GetMsgBody(container, message.AsBytes))).Message
.Headers.MessageId.ToString();
BrokerTracing.TraceVerbose(
"[AzureStorageTool] .CheckRequestQueue: queueName = {0}, cloudMessageId = {1}, messageId = {2}",
pendingQueue.Name,
message.Id,
messageId);
var query = new TableQuery <TableEntity>().Where("MessageId eq '" + messageId + "'");
var list = responseTable.ExecuteQuery(query).ToList();
if (list.Count > 0)
{
await pendingQueue.DeleteMessageAsync(await pendingQueue.GetMessageAsync());
}
else
{
// Add msg to request queue & delete it from pending queue.
message = await pendingQueue.GetMessageAsync();
await requestQueue.AddMessageAsync(new CloudQueueMessage(message.AsBytes));
await pendingQueue.DeleteMessageAsync(message);
BrokerTracing.TraceVerbose(
"[AzureStorageTool] .CheckRequestQueue: messageId = {0} is restored into request queue.",
messageId);
}
}
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureStorageTool] .CheckRequestQueue: queueName={0}, responseTable={1}, the exception, {2}",
pendingQueue.Name,
responseTable.Name,
e);
throw;
}
await requestQueue.FetchAttributesAsync();
}
/// <summary>
/// Gets the instance of scheduler adapter client
/// </summary>
/// <returns>returns the instance of the scheduler adapter client</returns>
public async Task <ISchedulerAdapter> GetSchedulerAdapterClientAsync()
{
// this.CheckClient();
bool newClientCreated = false;
ICommunicationObject client = this.schedulerAdapterClient as ICommunicationObject;
if (this.schedulerAdapterClient == null || client == null || client.State == CommunicationState.Faulted)
{
await this.createClientSS.WaitAsync();
try
{
if (this.schedulerAdapterClient == null)
{
await this.CreateClient();
if (!SoaCommonConfig.WithoutSessionLayer)
{
newClientCreated = true;
}
}
else
{
client = this.schedulerAdapterClient as ICommunicationObject;
if (client == null || client.State == CommunicationState.Faulted)
{
try
{
Utility.AsyncCloseICommunicationObject(client);
await this.CreateClient();
if (!SoaCommonConfig.WithoutSessionLayer)
{
newClientCreated = true;
}
}
catch (Exception e)
{
// Swallow exception when creating client
BrokerTracing.TraceError("[SchedulerAdapterClientFactory] Exception thrown when creating client: {0}", e);
}
}
}
}
finally
{
this.createClientSS.Release();
}
if (newClientCreated)
{
await this.monitor.RegisterJob();
}
}
return(this.schedulerAdapterClient);
}
/// <summary>
/// Callback method of the message retriever.
/// </summary>
/// <param name="messages">a collection of messages</param>
private void HandleMessages(IEnumerable <CloudQueueMessage> messages)
{
try
{
this.requestCache.Enqueue(messages);
this.semaphoreForRequest.Release();
}
catch (Exception e)
{
BrokerTracing.TraceError(SoaHelper.CreateTraceMessage("Proxy", "HandleMessages", string.Format("Error occurs, {0}", e)));
}
}
/// <summary>
/// Callback method of the timer.
/// </summary>
/// <param name="state">state object</param>
private void TimerCallback(object state)
{
try
{
this.Cleanup().GetAwaiter().GetResult();
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureStorageCleaner].TimerCallback: Cleanup failed, {0}",
e);
}
}
/// <summary>
/// EndOfMessage arrives
/// </summary>
/// <param name="msgCount">indicating the msg count</param>
/// <param name="timeout">indicating the timeout</param>
public void EndOfMessage(long msgCount, int batchId, int timeout)
{
lock (this.lockState)
{
if (this.state != BrokerClientState.ClientConnected)
{
BrokerTracing.EtwTrace.LogBrokerClientRejectEOM(this.sharedData.BrokerInfo.SessionId, this.clientId, this.state.ToString());
switch (this.state)
{
case BrokerClientState.GetResponse:
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_EOMReject_GetResponse, SR.EOMReject_GetResponse);
break;
case BrokerClientState.EndRequests:
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_EOMReject_EndRequests, SR.EOMReject_EndRequests);
break;
default:
ThrowHelper.ThrowSessionFault(SOAFaultCode.Broker_EOMRejected, SR.EOMRejected, this.state.ToString());
break;
}
}
// Stop the timer
this.timeoutManager.Stop();
try
{
this.Flush(msgCount, batchId, timeout, true);
}
catch (Exception e)
{
BrokerTracing.TraceError("[BrokerClient] Failed to flush messages: {0}", e);
// If EndRequests throws exception from broker queue, the
// timer is closed and never reactivated without user action.
// Thus, the broker client would leak forever and the broker
// process would run away if user does not purge the client
// or close the session later.
this.timeoutManager.RegisterTimeout(this.sharedData.Config.Monitor.ClientIdleTimeout, this.TimeoutToDisconnected, this.state);
throw;
}
// State will change if flush succeeded
// If flush failed, the state won't change
BrokerTracing.TraceInfo("[BrokerClient] Client {0}: State: ClientConnected ==> EndOfMessage", this.clientId);
this.state = BrokerClientState.EndRequests;
this.endOfMessageCalled = true;
}
}
/// <summary>
/// Handle fatal broker queue exception
/// </summary>
/// <param name="sender">indicating the sender</param>
/// <param name="fatalExceptionEventArgs">indicating the exception</param>
private void Queue_OnFatalExceptionEvent(object sender, ExceptionEventArgs fatalExceptionEventArgs)
{
BrokerTracing.TraceEvent(System.Diagnostics.TraceEventType.Critical, 0, "Fatal exception encountered. Exception = {0}", fatalExceptionEventArgs.Exception);
BrokerQueueException bqException = fatalExceptionEventArgs.Exception as BrokerQueueException;
if (bqException != null && bqException.ErrorCode == (int)BrokerQueueErrorCode.E_BQ_PERSIST_STORAGE_INSUFFICIENT)
{
this.monitor.FailServiceJob("Insufficient broker queue storage").GetAwaiter().GetResult();
}
else
{
BrokerTracing.TraceError("Unknown exception: {0}", fatalExceptionEventArgs);
}
}
public static async Task <CloudQueue> CreateQueueAsync(
string connectString,
string sessionId,
string queueName,
string clientId,
bool isRequest,
string note)
{
var info = new QueueInfo(sessionId, queueName, clientId, isRequest, note);
try
{
var table = GetTableClient(connectString).GetTableReference(queueTableName);
await table.CreateIfNotExistsAsync();
var insertOperation = TableOperation.Insert(info);
await table.ExecuteAsync(insertOperation);
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureStorageTool] .CreateQueueAsync: queueName={0} create info to table failed, the exception, {1}",
queueName,
e);
throw;
}
try
{
var queue = GetQueueClient(connectString).GetQueueReference(queueName);
if (await queue.CreateIfNotExistsAsync())
{
return(queue);
}
BrokerTracing.TraceWarning(
"[AzureStorageTool] .CreateQueueAsync: queueName={0} has existed.",
queueName);
throw new Exception("Queue with the queueName has existed.");
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureStorageTool] .CreateQueueAsync: queueName={0} create queue failed, the exception, {1}",
queueName,
e);
throw;
}
}
/// <summary>
/// Callback of the CloudQueue.BeginAddMessage method.
/// </summary>
/// <param name="ar">async result</param>
/// <remarks>
/// Notice: This method doesn't throw exception. It invokes callback
/// and pass exception to it in case exception occurs.
/// </remarks>
private void BeginAddMessageCallback(IAsyncResult ar)
{
BrokerTracing.TraceVerbose("[AzureServiceClient].BeginAddMessageCallback: Enter callback method of BeginAddMessage.");
var reliableState = ar.AsyncState as ReliableQueueClient.ReliableState;
QueueAsyncResult asyncResult = reliableState.State as QueueAsyncResult;
Debug.Assert(asyncResult != null, "reliableState.State must be a QueueAsyncResult.");
try
{
BrokerTracing.TraceVerbose(
"[AzureServiceClient].BeginAddMessageCallback: Try to complete adding message {0}",
asyncResult.MessageId);
asyncResult.StorageClient.EndAddMessage(ar);
}
catch (StorageException e)
{
BrokerTracing.TraceError(
"[AzureServiceClient].BeginAddMessageCallback: Failed to complete adding message {0}, {1}",
asyncResult.MessageId,
e.ToString());
if (BurstUtility.IsQueueNotFound(e))
{
// StorageException happens here when want to add request
// messages, so it must be request queue not found. Handle
// the outstanding messages, which are already sent to
// request queue, but maybe not got by proxy. And should
// consider the multi request queue case when there are
// multi azure deployments.
this.manager.HandleInvalidRequestQueue(new RequestStorageException(e), this.requestStorageClient.QueueName);
}
this.manager.CompleteCallback(asyncResult, null, new RequestStorageException(e));
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureServiceClient].BeginAddMessageCallback: Failed to complete adding message {0}, {1}",
asyncResult.MessageId,
e.ToString());
this.manager.CompleteCallback(asyncResult, null, e);
}
}
/// <summary>
/// Handle client failure
/// </summary>
/// <param name="client">targeted client</param>
private static void HandleClientFailure(AzureServiceClient client)
{
try
{
BrokerTracing.TraceWarning(
"[AzureDispatcher] HandleClientFailure: Handle invalid client, {0}, {1}",
client,
client.ServiceClient.Endpoint.Address);
Dictionary <EndpointAddress, ProxyClientPool> .ValueCollection pools;
lock (LockProxyClientPoolDic)
{
pools = ProxyClientPoolDic.Values;
}
foreach (var pool in pools)
{
bool existInPool = false;
lock (pool)
{
BrokerTracing.TraceVerbose(
"[AzureDispatcher] HandleClientFailure: Remove client {0} from pool.", client);
existInPool = pool.RemoveProxyClient(client);
}
if (existInPool)
{
BrokerTracing.TraceVerbose(
"[AzureDispatcher] HandleClientFailure: Close client {0}", client);
// Close the proxy client if any exception is encountered.
// As a result, async pending callback on clients will be
// invoked (with exception).
client.AsyncClose();
return;
}
}
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureDispatcher] HandleClientFailure: Error occurs, {0}", e);
}
}
/// <summary>
/// Refresh the client.
/// For on-premise cluster, just create a new client.
/// </summary>
/// <param name="clientIndex">client index</param>
/// <param name="exceptionIndirect">
/// it is false for the on-premise cluster
/// </param>
/// <param name="messageId">message Id</param>
/// <returns>should increase the retry count or not</returns>
public override async Task <bool> RefreshClientAsync(int clientIndex, bool exceptionIndirect, Guid messageId)
{
BrokerTracing.TraceError(
BrokerTracing.GenerateTraceString(
"OnPremiseRequestSender",
"RefreshClientAsync",
this.TaskId,
clientIndex,
this.Client.ToString(),
messageId,
string.Format("exceptionIndirect = {0}", exceptionIndirect)));
await this.CreateClientAsync(false, clientIndex).ConfigureAwait(false);
return(true);
}
/// <summary>
/// the putting request complete thread proc
/// </summary>
/// <param name="state">the thread pool callback state</param>
private static void PutRequestComplete(PutRequestCallback callback, object state)
{
if (callback == null)
{
return;
}
try
{
callback(null, state);
}
catch (Exception e)
{
BrokerTracing.TraceError("[MemoryPersist] .PutRequestComplete: callback failed, Exception:{0}.", e.ToString());
}
}
/// <summary>
/// Trace error level log.
/// </summary>
/// <param name="className">class name</param>
/// <param name="methodName">method name</param>
/// <param name="format">message format</param>
/// <param name="args">arguments of message format</param>
public static void TraceError(string className, string methodName, string format, params object[] args)
{
string traceMessage = SoaHelper.CreateTraceMessage(className, methodName, format, args);
#if PROXY
Microsoft.Hpc.BrokerProxy.AzureBrokerProxyTrace.TraceError(traceMessage);
#elif DATAPROXY
Microsoft.Hpc.Scheduler.Session.Data.Internal.DataProxyTrace.TraceError(traceMessage);
#elif DATASVC
Microsoft.Hpc.Scheduler.Session.Data.Internal.DataServiceTraceHelper.TraceEvent(TraceEventType.Error, traceMessage);
#elif CONSOLE
Console.Error.WriteLine(string.Format("{0} - {1}", DateTime.Now, traceMessage));
#else
BrokerTracing.TraceError(traceMessage);
#endif
}
/// <summary>
/// the putting response complete thread proc
/// </summary>
/// <param name="state">the thread pool callback state</param>
private static void PutResponseComplete(int responseCount, int faultResponseCount, bool isLastResponse, PutResponseCallback putResponseCallback, object callbackState)
{
if (putResponseCallback == null)
{
return;
}
try
{
putResponseCallback(null, responseCount, faultResponseCount, isLastResponse, null, callbackState);
}
catch (Exception e)
{
BrokerTracing.TraceError("[MemoryPersist] .PutResponseComplete: callback failed, Exception:{0}.", e.ToString());
}
}
/// <summary>
/// Async method of ProcessMessage.
/// </summary>
/// <param name="request">request message</param>
/// <param name="callback">callback method</param>
/// <param name="asyncState">async state</param>
/// <returns>async result</returns>
public IAsyncResult BeginProcessMessage(Message request, AsyncCallback callback, object asyncState)
{
MessageBuffer buffer = request.CreateBufferedCopy(int.MaxValue);
byte[] messageData = AzureQueueItem.Serialize(buffer.CreateMessage());
QueueAsyncResult asyncResult = new QueueAsyncResult(callback, asyncState);
UniqueId messageId = request.Headers.MessageId;
asyncResult.MessageId = messageId;
asyncResult.StorageClient = this.requestStorageClient;
try
{
BrokerTracing.TraceVerbose(
"[AzureServiceClient].BeginProcessMessage: Try to add message {0} to the request queue.",
messageId);
var reliableState = new ReliableQueueClient.ReliableState(asyncResult, messageId);
// Notice: It can happen that response message is back before
// EndAddMessage is called on the request message. So
// add/update the callback info to AzureQueueManager before
// calling BeginAddMessage avoid the issue that response comes
// back but can't find callback info.
this.manager.AddQueueAsyncResult(asyncResult, this.requestStorageClient.QueueName, this.responseStorageName);
this.requestStorageClient.BeginAddMessage(messageData, messageId, this.BeginAddMessageCallback, reliableState);
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureServiceClient].BeginProcessMessage: Failed to add message {0}, {1}",
messageId,
e);
this.manager.CompleteCallback(asyncResult, null, e);
}
finally
{
buffer.Close();
}
return(asyncResult);
}
/// <summary>
/// Service instance is failed
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void ServiceInstanceFailed(object sender, ServiceInstanceFailedEventArgs e)
{
Dispatcher dispatcher = sender as Dispatcher;
DispatcherInfo dispatcherInfo = dispatcher.Info;
// Filter out duplicate failures
lock (this.lockFailedDispatcherList)
{
if (this.failedDispatcherList.Contains(dispatcher.TaskId))
{
return;
}
this.failedDispatcherList.Add(dispatcher.TaskId);
}
BrokerTracing.TraceError("[DispatcherManager] Service instance failed! Task id = {0}, node name = {1}", dispatcherInfo.UniqueId, dispatcherInfo.MachineName);
// If the service is unavailable, just block the dispatcher. We cannot blacklist it because it may have just been preempted and the node is still good.
// Furthermore the network may be temporarily out but the CN and its app install is fine
if (e.Fault.Code == (int)SOAFaultCode.Service_Unreachable)
{
this.BlockDispatcher(dispatcherInfo);
}
// If the service cannot be initialized, remove and blacklist the dispatcher. We know the service cannot be loaded
else if (e.Fault.Code == (int)SOAFaultCode.Service_InitializeFailed)
{
if (this.RemoveDispatcher(dispatcherInfo.UniqueId, /*exitServiceHost =*/ false, false))
{
// Should use the machine virtual name for scheduler API to exclude a node.
this.monitor.BlacklistNode(SoaHelper.IsOnAzure() ? dispatcherInfo.MachineVirtualName : dispatcherInfo.MachineName).GetAwaiter().GetResult();
}
}
// If the service host is preempted, remove the dispatcher.
else if (e.Fault.Code == SOAFaultCode.Service_Preempted)
{
this.RemoveDispatcher(dispatcherInfo.UniqueId, true, true);
}
// There should be no other possible failure codes
else
{
Debug.Assert(false, String.Format("Invalid fault code sent to ServiceInstanceFailed - {0}", e.Fault.Code));
}
}
public static async Task <CloudTable> CreateTableAsync(
string connectString,
string sessionId,
string tableName,
string clientId,
bool isRequest,
string note)
{
var info = new QueueInfo(sessionId, tableName, clientId, isRequest, note);
try
{
var table = GetTableClient(connectString).GetTableReference(queueTableName);
await table.CreateIfNotExistsAsync();
var insertOperation = TableOperation.Insert(info);
await table.ExecuteAsync(insertOperation);
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureStorageTool] .CreateTableAsync: Table={0} insert info in storage table failed, the exception, {1}",
tableName,
e);
throw;
}
try
{
var responseTable = GetTableClient(connectString).GetTableReference(tableName);
if (await responseTable.CreateIfNotExistsAsync())
{
return(responseTable);
}
throw new Exception("Table with the tableName has existed.");
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureStorageTool] .CreateTableAsync: Table={0} create error, the exception, {1}",
tableName,
e);
throw;
}
}