/// <summary>
/// Add QueueAsyncResult to the dictionary.
/// </summary>
/// <param name="result">async result</param>
/// <param name="requestQueueName">request queue name</param>
/// <param name="responseQueueName">response queue name</param>
public void AddQueueAsyncResult(QueueAsyncResult result, string requestQueueName, string responseQueueName)
{
ResponseStorageException e;
this.responseQueueNotFound.TryGetValue(responseQueueName, out e);
if (e == null)
{
BrokerTracing.TraceVerbose(
"[AzureQueueManager].AddQueueAsyncResult: Add QueueAsyncResult {0} to the dictionary.",
result.MessageId);
this.callbacks.AddOrUpdate(result.MessageId, result, (key, value) => result);
this.requestsMappingToRequestQueue.AddOrUpdate(result.MessageId, requestQueueName, (key, value) => requestQueueName);
this.requestsMappingToResponseQueue.AddOrUpdate(result.MessageId, responseQueueName, (key, value) => responseQueueName);
}
else
{
BrokerTracing.TraceError(
"[AzureQueueManager].AddQueueAsyncResult: Response queue is not found, {0}",
responseQueueName);
throw e;
}
}
/// <summary>
/// Remove QueueAsyncResult from the dictionary.
/// </summary>
/// <param name="messageId">message Id</param>
public void RemoveQueueAsyncResult(UniqueId messageId)
{
string value;
if (!this.requestsMappingToRequestQueue.TryRemove(messageId, out value))
{
BrokerTracing.TraceError(
"[AzureQueueManager].CompleteCallback: Failed to remove message Id {0} from requestsMappingToRequestQueue.",
messageId);
}
if (!this.requestsMappingToResponseQueue.TryRemove(messageId, out value))
{
BrokerTracing.TraceError(
"[AzureQueueManager].CompleteCallback: Failed to remove message Id {0} from requestsMappingToResponseQueue.",
messageId);
}
QueueAsyncResult result;
if (!this.callbacks.TryRemove(messageId, out result))
{
BrokerTracing.TraceError(
"[AzureQueueManager].CompleteCallback: Failed to remove QueueAsyncResult {0} from callbacks.",
messageId);
}
else
{
BrokerTracing.TraceVerbose(
"[AzureQueueManager].CompleteCallback: Remove QueueAsyncResult {0} from callbacks.",
messageId);
}
}
/// <summary>
/// Create request queue for specified Azure service.
/// </summary>
/// <param name="azureServiceName">azure service name</param>
public void CreateRequestStorage(string azureServiceName)
{
this.requestStorage.GetOrAdd(
azureServiceName,
(key) =>
{
string requestStorageName = SoaHelper.GetRequestStorageName(this.clusterId.ToString(), key);
BrokerTracing.TraceVerbose(
"[AzureQueueManager].CreateRequestStorage: Try to create the request storage {0} for Azure service {1}",
requestStorageName,
key);
this.CreateStorageClient(RetryPolicyForRequestStorage);
CloudQueue queue = this.queueClient.GetQueueReference(requestStorageName);
AzureQueueManager.CreateQueueWithRetry(queue);
CloudBlobContainer container = this.blobClient.GetContainerReference(requestStorageName);
AzureQueueManager.CreateContainerWithRetry(container);
if (Interlocked.CompareExchange(ref this.requestQueueExist, 1, 0) == 0)
{
BrokerTracing.EtwTrace.LogQueueCreatedOrExist(this.sessionId, requestStorageName);
}
return(new Tuple <CloudQueue, CloudBlobContainer>(queue, container));
});
}
/// <summary>
/// Create cloud queue/blob client. Don't create it in class
/// constructor in order to delay retrieve the connection string.
/// </summary>
/// <param name="retryPolicy">
/// It is storage retry policy. We have different policy for request
/// storage and response storage.
/// </param>
private void CreateStorageClient(IRetryPolicy retryPolicy)
{
if (this.queueClient == null || this.blobClient == null)
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(this.storageConnectionString);
if (this.queueClient == null)
{
BrokerTracing.TraceVerbose(
"[ResponseQueueManager].CreateStorageClient: Try to create the response queue client {0}",
this.responseStorageName);
CloudQueueClient tmpQueueClient = storageAccount.CreateCloudQueueClient();
tmpQueueClient.DefaultRequestOptions.RetryPolicy = retryPolicy;
Interlocked.CompareExchange <CloudQueueClient>(ref this.queueClient, tmpQueueClient, null);
}
if (this.blobClient == null)
{
BrokerTracing.TraceVerbose(
"[ResponseQueueManager].CreateStorageClient: Try to create the response container client {0}",
this.responseStorageName);
CloudBlobClient tmpBlobClient = storageAccount.CreateCloudBlobClient();
tmpBlobClient.DefaultRequestOptions.RetryPolicy = retryPolicy;
Interlocked.CompareExchange <CloudBlobClient>(ref this.blobClient, tmpBlobClient, null);
}
}
}
private void EndExit(IAsyncResult result)
{
var client = (ProxyServiceControlClient)result.AsyncState;
bool needRetry = false;
try
{
BrokerTracing.TraceVerbose("[AzureServiceHostController].EndExit: Try to call EndExit method, https = {0}", this.https);
client.EndExit(result);
}
catch (Exception e)
{
BrokerTracing.TraceWarning("[AzureServiceHostController].EndExit: Exception thrown while exiting service host: {0}", e);
if (++this.retryCount <= MaxExitServiceRetryCount)
{
needRetry = true;
}
else
{
this.OnFailed();
}
}
finally
{
Utility.AsyncCloseICommunicationObject(client);
this.controllerClient = null;
}
if (needRetry)
{
this.BeginExit(this.OnFailed);
}
}
/// <summary>
/// Get a service client from pool and make sure it is workable.
/// </summary>
/// <remarks>
/// Notice: Call ReleaseProxyClient when you're done with ServiceClient.
/// </remarks>
/// <returns>service client</returns>
public AzureServiceClient GetProxyClient()
{
while (true)
{
AzureServiceClient client = this.InternalGetProxyClient();
if (client.ServiceClient.State == CommunicationState.Closed ||
client.ServiceClient.State == CommunicationState.Closing ||
client.ServiceClient.State == CommunicationState.Faulted)
{
BrokerTracing.TraceVerbose(
"[ProxyClientPool].GetProxyClient: Client is not ready for use, remove it, {0}, {1}",
client,
client.ServiceClient.Endpoint.Address);
this.RemoveProxyClient(client);
if (client.ServiceClient.State == CommunicationState.Faulted)
{
client.AsyncClose();
}
}
else
{
BrokerTracing.TraceVerbose(
"[ProxyClientPool].GetProxyClient: Get a client ready for use, {0}, {1}",
client,
client.ServiceClient.Endpoint.Address);
return(client);
}
}
}
/// <summary>
/// Async open service host.
/// </summary>
/// <param name="num"></param>
/// <param name="taskInfo"></param>
/// <param name="regPath"></param>
/// <param name="svcName"></param>
/// <returns></returns>
private static async Task <ServiceTaskDispatcherInfo> OpenSvcHostAsync(
string sessionId,
ServiceTaskDispatcherInfo taskDispatcherInfo,
string regPath,
string svcName,
Version svcVersion,
Dictionary <string, string> environment,
Dictionary <string, string> dependFilesInfo)
{
var invalidList = invalidIds;
if (invalidList.Contains(taskDispatcherInfo.TaskId))
{
return(null);
}
string fileName = SoaRegistrationAuxModule.GetRegistrationFileName(svcName, svcVersion);
BrokerTracing.TraceVerbose("[SvcHostHttpClient] Started send request, taskId is {0} and session id is {1}", taskDispatcherInfo.TaskId, sessionId);
// HTTP POST
var serviceInfo = new ServiceInfo(sessionId, taskDispatcherInfo.TaskId, taskDispatcherInfo.FirstCoreId, regPath + "\\", fileName, environment, dependFilesInfo);
var result = await SvcHostHttpClient.PostAsJsonAsync <ServiceInfo>(new Uri($"{Prefix}{taskDispatcherInfo.MachineName}:{Port}/{EndPointName}/api/{ApiName}"), serviceInfo);
BrokerTracing.TraceVerbose("[OpenSvcHost].result:{0}", result);
result.EnsureSuccessStatusCode();
return(taskDispatcherInfo);
}
/// <summary>
/// Add a dispatcher to blocked dispatcher list
/// </summary>
/// <param name="dispatcherInfo">dispatcher info</param>
private void AddBlockedDispatcher(DispatcherInfo dispatcherInfo)
{
try
{
dispatcherInfo.BlockTime = DateTime.Now;
this.blockedDispatcherDic.Add(dispatcherInfo.UniqueId, dispatcherInfo);
// if a dispatcher has never been blocked, take it as "young" blocked disaptcher.
if (dispatcherInfo.BlockRetryCount <= 0)
{
BrokerTracing.TraceVerbose("[DispatcherManager] Increment youngBlockedDispatcherCount, task id={0}, BlockRetryCount={1}", dispatcherInfo.UniqueId, dispatcherInfo.BlockRetryCount);
this.youngBlockedDispatcherCount++;
}
this.blockedDispatcherQueue.Enqueue(dispatcherInfo);
if (this.blockedDispatcherQueue.Count == 1)
{
BrokerTracing.TraceVerbose("[DispatcherManager] Block dispatcher: change unblock timer, task id = {0}", dispatcherInfo.UniqueId);
this.unblockTimer.Change(this.blockTimeSpan, TimeSpan.FromMilliseconds(-1));
}
BrokerTracing.TraceInfo("[DispatcherManager] Add dispatcher {0} into the blocked dispatcher list.", dispatcherInfo.UniqueId);
}
catch (ArgumentException)
{
BrokerTracing.TraceError("[DispatcherManager] Dispatcher {0} already exist in the blocked dispatcher list.", dispatcherInfo.UniqueId);
}
}
/// <summary>
/// Instantiate a new instance of MSMQMessageFetcher class.
/// </summary>
/// <param name="messageQueue">target MSMQ queue</param>
/// <param name="messageCount">number of messages in queue</param>
/// <param name="messageFormatter">message formatter for deserializing MSMQ message</param>
/// <param name="prefetchCacheCapacity">prefetch cache capacity</param>
/// <param name="maxOutstandingFetchCount">maximun number of outstanding BeginPeek operations</param>
public MSMQMessageFetcher(MessageQueue messageQueue, long messageCount, IMessageFormatter messageFormatter, int prefetchCacheCapacity, int maxOutstandingFetchCount)
{
this.msmqQueueField = messageQueue;
this.msmqMessageCount = messageCount;
this.messageFormatter = messageFormatter;
this.prefetchCacheCapacity = prefetchCacheCapacity;
this.maxOutstandingFetchCount = maxOutstandingFetchCount;
this.prefetchCredit = this.prefetchCacheCapacity;
this.messagePeekCursorField = new RefCountedCursor(this.msmqQueueField.CreateCursor());
this.messagePeekActionField = PeekAction.Current;
this.msmqQueueField.Disposed += this.OnQueueDisposed;
this.prefetchTimer.AutoReset = false;
this.prefetchTimer.Interval = 500;
this.prefetchTimer.Elapsed += (sender, args) =>
{
Debug.WriteLine("[MSMQMessageFetcher] .prefetchTimer raised.");
this.PeekMessage();
if (!this.isDisposedField)
{
this.prefetchTimer.Enabled = true;
}
};
this.prefetchTimer.Enabled = true;
BrokerTracing.TraceVerbose("[MSMQMessageFetcher] .Create new instance: prefetchCacheCapacity={0}, maxOutstandingFetchCount={1}", this.prefetchCacheCapacity, this.maxOutstandingFetchCount);
}
/// <summary>
/// Initializes a new instance of the HpcSchedulerAdapterInternalClient class
/// </summary>
/// <param name="headNodeMachine">indicating the headnode</param>
public HpcSchedulerAdapterInternalClient(string headNodeMachine, string certThumbprint)
: base(
BindingHelper.HardCodedInternalSchedulerDelegationBinding,
SoaHelper.CreateInternalCertEndpointAddress(new Uri(SoaHelper.GetSchedulerDelegationInternalAddress(headNodeMachine)), certThumbprint))
{
#if BrokerLauncher
BrokerTracing.TraceVerbose("[HpcSchedulerAdapterInternalClient] In constructor");
#endif
// use certificate for cluster internal authentication
this.ClientCredentials.UseInternalAuthentication(certThumbprint);
if (BrokerIdentity.IsHAMode)
{
// Bug 10301 : Explicitly open channel when impersonating the resource group's account if running on failover cluster so identity flows correctly when
// calling HpcSession.
// NOTE: The patch we got from the WCF team (KB981001) only works when the caller is on a threadpool thread.
// NOTE: Channel must be opened before setting OperationTimeout
using (BrokerIdentity identity = new BrokerIdentity())
{
identity.Impersonate();
this.Open();
}
}
this.InnerChannel.OperationTimeout = OperationTimeout;
}
/// <summary>
/// Block a dispatcher temporarily
/// </summary>
/// <param name="dispatcherInfo">dispatcher info</param>
public void BlockDispatcher(DispatcherInfo dispatcherInfo)
{
Dispatcher dispatcher = null;
int activeDispatcherCount = 0;
lock (this.lockThis)
{
dispatcher = this.RemoveActiveDispatcher(dispatcherInfo.UniqueId);
if (dispatcher == null)
{
BrokerTracing.TraceVerbose("[DispatcherManager] Block dispatcher failed: {0} is not active", dispatcherInfo.UniqueId);
return;
}
this.AddBlockedDispatcher(dispatcherInfo);
// Note: dispatchers that are blocked but not retried are also considered as 'active'
activeDispatcherCount = this.sharedData.DispatcherCount + this.youngBlockedDispatcherCount;
}
dispatcher.Close();
// check job healthy to see if number of active dispatchers is smaller than (job.minResourceUnit)
this.monitor.CheckJobHealthy(activeDispatcherCount);
}
/// <summary>
/// Create response queue/blob container for the specified session.
/// </summary>
private void CreateResponseStorage()
{
this.CreateStorageClient(RetryPolicyForResponseStorage);
if (this.responseQueue == null)
{
BrokerTracing.TraceVerbose(
"[ResponseQueueManager].CreateResponseStorage: Try to create the response queue {0}",
this.responseStorageName);
CloudQueue queue = this.queueClient.GetQueueReference(this.responseStorageName);
AzureQueueManager.CreateQueueWithRetry(queue);
if (Interlocked.CompareExchange <CloudQueue>(ref this.responseQueue, queue, null) == null)
{
BrokerTracing.EtwTrace.LogQueueCreatedOrExist(this.sessionId, this.responseStorageName);
}
}
if (this.responseContainer == null)
{
BrokerTracing.TraceVerbose(
"[ResponseQueueManager].CreateResponseStorage: Try to create the response container {0}",
this.responseStorageName);
CloudBlobContainer container = this.blobClient.GetContainerReference(this.responseStorageName);
AzureQueueManager.CreateContainerWithRetry(container);
Interlocked.CompareExchange <CloudBlobContainer>(ref this.responseContainer, container, null);
}
}
/// <summary>
/// Get ProxyClientPool according to broker proxy endpoint address.
/// </summary>
/// <param name="proxyEpr">broker proxy endpoint address</param>
/// <param name="proxyClientPoolCapacity"> capacity of ProxyClient pool. It is used when creating a new ProxyClientPool instance</param>
/// <returns>ProxyClientPool instance</returns>
private ProxyClientPool AttachProxyClientPool(EndpointAddress proxyEpr, int proxyClientPoolCapacity)
{
ProxyClientPool clientPool = null;
bool createFlag = false;
lock (LockProxyClientPoolDic)
{
// if there is already a connection pool to the target broker proxy, reuse it; otherwise, create a new one.
if (!ProxyClientPoolDic.TryGetValue(proxyEpr, out clientPool))
{
clientPool = new ProxyClientPool(proxyEpr, proxyClientPoolCapacity);
clientPool.RefCount = 1;
ProxyClientPoolDic.Add(proxyEpr, clientPool);
createFlag = true;
}
else
{
clientPool.RefCount++;
}
}
if (createFlag)
{
BrokerTracing.TraceVerbose("[AzureDispatcher] New ProxyClientPool instance is created. Proxy EPR address = {0}, capacity = {1}", proxyEpr, proxyClientPoolCapacity);
}
return(clientPool);
}
protected void CheckAndGetMoreMessages()
{
while (!this.isDisposedField)
{
BrokerTracing.TraceVerbose(
"[AzureQueueMessageFetcher] .CheckAndGetMoreMessages: fetcherId={0} prefetchCredit={1}, outstandingFetchCount={2}, msmqMessageCount={3}, pendingFetchCount={4}",
this.fetcherId,
this.prefetchCredit,
this.outstandingFetchCount,
this.messageCount,
this.pendingFetchCount);
int increasedFetchCount = 0;
while (this.AddFetchCount())
{
++increasedFetchCount;
Interlocked.Increment(ref this.pendingFetchCount);
}
BrokerTracing.TraceVerbose(
"[AzureQueueMessageFetcher] .CheckAndGetMoreMessages: fetcherId={0} increasedFetchCount={1}",
this.fetcherId,
increasedFetchCount);
return;
}
}
/// <summary>
/// Initializes a new instance of the HpcSchedulerAdapterClient class
/// </summary>
/// <param name="headnode">indicating the headnode</param>
/// <param name="instanceContext">indicating the instance context</param>
public HpcSchedulerAdapterClient(string headnode, string certThrumbprint, InstanceContext instanceContext)
: base(
instanceContext,
BindingHelper.HardCodedInternalSchedulerDelegationBinding,
SoaHelper.CreateInternalCertEndpointAddress(new Uri(SoaHelper.GetSchedulerDelegationAddress(headnode)), certThrumbprint))
{
BrokerTracing.TraceVerbose("[HpcSchedulerAdapterClient] In constructor");
this.ClientCredentials.UseInternalAuthentication(certThrumbprint);
if (BrokerIdentity.IsHAMode)
{
// Bug 10301 : Explicitly open channel when impersonating the resource group's account if running on failover cluster so identity flows correctly when
// calling HpcSession.
// NOTE: The patch we got from the WCF team (KB981001) only works when the caller is on a threadpool thread.
// NOTE: Channel must be opened before setting OperationTimeout
using (BrokerIdentity identity = new BrokerIdentity())
{
identity.Impersonate();
this.Open();
}
}
this.InnerChannel.OperationTimeout = SchedulerAdapterTimeout;
foreach (OperationDescription op in this.Endpoint.Contract.Operations)
{
DataContractSerializerOperationBehavior dataContractBehavior = op.Behaviors.Find <DataContractSerializerOperationBehavior>() as DataContractSerializerOperationBehavior;
if (dataContractBehavior != null)
{
dataContractBehavior.MaxItemsInObjectGraph = int.MaxValue;
}
}
}
/// <summary>
/// If the exception is CommunicationException or from the connection between the proxy
/// and host, it might be caused by the job preemption or cancellation
/// </summary>
/// <param name="e">exception received by azure dispatcher</param>
/// <returns>should check the error code or not</returns>
protected override bool ShouldCheckTaskErrorCode(Exception e)
{
BrokerTracing.TraceVerbose("[AzureDispatcher].ShouldCheckTaskErrorCode e: {0}, e.Source: {1}", e, e.Source);
return(e is CommunicationException &&
e.Source == IndirectExceptionMark &&
this.SharedData.ServiceConfig.EnableMessageLevelPreemption);
}
/// <summary>
/// Gets the enpoint address for a service host
/// </summary>
/// <param name="isController">indicating whether controller address is required</param>
/// <returns>endpoint address</returns>
protected override EndpointAddress GetEndpointAddress(bool isController)
{
string hostnameWithPrefix = this.MachineName;
int port = PortHelper.ConvertToPort(this.firstCoreId, isController);
if (!String.IsNullOrEmpty(this.networkPrefix))
{
hostnameWithPrefix = string.Format(CultureInfo.InvariantCulture, "{0}.{1}", this.networkPrefix, hostnameWithPrefix);
}
string epr = BindingHelper.GenerateServiceHostEndpointAddress(hostnameWithPrefix, this.jobId, this.taskId, port, this.isHttp) + Constant.ServiceHostEndpointPath;
if (isController)
{
epr += Constant.ServiceHostControllerEndpointPath;
BrokerTracing.TraceVerbose("[WssDispatcherInfo]. Service host controller EPR = {0}.", epr);
}
else
{
BrokerTracing.TraceVerbose("[WssDispatcherInfo]. Service host EPR = {0}.", epr);
}
return(new EndpointAddress(
new Uri(epr),
EndpointIdentity.CreateDnsIdentity(Constant.HpcWssServiceIdentity),
new AddressHeaderCollection()
));
}
public virtual void BeginExit(Action onFailed)
{
try
{
this.OnFailed = onFailed;
BrokerTracing.TraceVerbose("[SerivceHostController].BeginExit: construct the client. Binding {0}, Endpoint {1}", this.binding, this.controllerEndpoint);
HpcServiceHostClient serviceHostClient = new HpcServiceHostClient(this.binding, this.controllerEndpoint, BrokerIdentity.IsHAMode);
this.controllerClient = serviceHostClient;
BrokerTracing.TraceVerbose("[SerivceHostController].BeginExit: BeginExit. Binding {0}, Endpoint {1}", this.binding, this.controllerEndpoint);
serviceHostClient.BeginExit(this.EndExit, serviceHostClient);
BrokerTracing.TraceVerbose("[SerivceHostController].BeginExit: Called. Binding {0}, Endpoint {1}", this.binding, this.controllerEndpoint);
}
catch (Exception ex)
{
BrokerTracing.TraceWarning("[SerivceHostController].BeginExit: Exception : {0}", ex);
this.OnFailed();
// If BeginExit fails, dont retry any further. The task likely terminated already
if (this.controllerClient != null)
{
Utility.AsyncCloseICommunicationObject(this.controllerClient);
}
}
}
/// <summary>
/// Mark all the dispatchers to stop
/// </summary>
private void StopAllDispatchers()
{
BrokerTracing.TraceVerbose("[GracefulPreemptionHandler].StopAllDispatchers: stop all dispatchers");
// Occupied too much resource, stop the prefetch to release.
this.dispatcherManager.StopAllDispatchers();
}
/// <summary>
/// Create a IService instance with specified index.
/// </summary>
/// <param name="getNextRequest">
/// trigger the client to retrieve next request
/// </param>
/// <param name="clientIndex">
/// index of the client
/// </param>
public virtual async Task CreateClientAsync(bool getNextRequest, int clientIndex)
{
if (this.IServiceClient != null)
{
BrokerTracing.TraceWarning(
BrokerTracing.GenerateTraceString(
"RequestSender",
"CreateClientAsync",
this.TaskId,
clientIndex,
this.IServiceClient.ToString(),
string.Empty,
"Closed former client proxy."));
this.CloseClient();
}
try
{
await this.CreateClientAsync().ConfigureAwait(false);
BrokerTracing.TraceVerbose(
BrokerTracing.GenerateTraceString(
"RequestSender",
"CreateClientAsync",
this.TaskId,
clientIndex,
this.IServiceClient.ToString(),
string.Empty,
"Created a new client."));
this.Dispatcher.PassBindingFlags[clientIndex] = true;
GetNextRequestState state = null;
if (getNextRequest)
{
state = new GetNextRequestState(this.Dispatcher.GetNextRequest, clientIndex);
}
this.StartClient(state);
}
catch (Exception e)
{
BrokerTracing.TraceEvent(
TraceEventType.Error,
0,
"[RequestSender] .CreateClientAsync: ID = {0}, init client failed: {1}",
this.TaskId,
e);
if (this.IServiceClient != null)
{
this.CloseClient();
}
this.Dispatcher.CloseThis();
}
}
/// <summary>
/// Dispose the instance
/// </summary>
protected override void DisposeInternal()
{
BrokerTracing.TraceVerbose("[AzureServiceClient]. DisposeInternal: Dispose client {0}.", this);
base.DisposeInternal();
this.AsyncClose();
}
/// <summary>
/// Triggered when a client is disconnected
/// </summary>
/// <param name="sender">indicating the client</param>
/// <param name="e">indicating the event args</param>
private void Client_ClientDisconnected(object sender, EventArgs e)
{
BrokerClient client = sender as BrokerClient;
Debug.Assert(client != null, "[BrokerClientManager] ClientDisconnected Event: Sender should be the BrokerClient type.");
BrokerTracing.TraceVerbose("[BrokerClientManager] Remove client {0}", client.ClientId);
this.RemoveClient(client, false);
}
public static async Task RestoreRequest(
CloudQueue requestQueue,
CloudQueue pendingQueue,
CloudTable responseTable,
CloudBlobContainer container)
{
try
{
while (true)
{
var message = await pendingQueue.PeekMessageAsync();
if (message == null)
{
break;
}
var messageId =
((BrokerQueueItem)formatter.Deserialize(await GetMsgBody(container, message.AsBytes))).Message
.Headers.MessageId.ToString();
BrokerTracing.TraceVerbose(
"[AzureStorageTool] .CheckRequestQueue: queueName = {0}, cloudMessageId = {1}, messageId = {2}",
pendingQueue.Name,
message.Id,
messageId);
var query = new TableQuery <TableEntity>().Where("MessageId eq '" + messageId + "'");
var list = responseTable.ExecuteQuery(query).ToList();
if (list.Count > 0)
{
await pendingQueue.DeleteMessageAsync(await pendingQueue.GetMessageAsync());
}
else
{
// Add msg to request queue & delete it from pending queue.
message = await pendingQueue.GetMessageAsync();
await requestQueue.AddMessageAsync(new CloudQueueMessage(message.AsBytes));
await pendingQueue.DeleteMessageAsync(message);
BrokerTracing.TraceVerbose(
"[AzureStorageTool] .CheckRequestQueue: messageId = {0} is restored into request queue.",
messageId);
}
}
}
catch (Exception e)
{
BrokerTracing.TraceError(
"[AzureStorageTool] .CheckRequestQueue: queueName={0}, responseTable={1}, the exception, {2}",
pendingQueue.Name,
responseTable.Name,
e);
throw;
}
await requestQueue.FetchAttributesAsync();
}
/// <summary>
/// Dispose this MSMQMessageFetcher instance.
/// </summary>
public void SafeDispose()
{
BrokerTracing.TraceVerbose("[MSMQMessageFetcher] .SafeDispose.");
//
// SafeDispose:
// 1. mark this instance as disposed
// 2. wait for all outstanding BeginPeek operations back, and then dispose the cursor.
//
if (!this.isDisposedField)
{
this.isDisposedField = true;
// dispose prefetched messages
this.lockPrefetchCache.EnterWriteLock();
try
{
if (this.prefetchCache != null)
{
// below line of code is put inside of prefetchCache lock scope just to ensure it's executed only once
this.msmqQueueField.Disposed -= this.OnQueueDisposed;
if (!this.prefetchCache.IsEmpty)
{
BrokerTracing.TraceWarning("[MSMQMessageFetcher] .SafeDispose: PrefetchCache is not empty when disposing.");
}
foreach (MessageResult result in this.prefetchCache)
{
if (result.Message != null)
{
result.Message.Dispose();
}
}
this.prefetchCache = null;
}
}
finally
{
this.lockPrefetchCache.ExitWriteLock();
}
this.rwlockMessagePeekCursorField.EnterWriteLock();
try
{
if (this.messagePeekCursorField != null)
{
this.messagePeekCursorField.Release();
this.messagePeekCursorField = null;
}
}
finally
{
this.rwlockMessagePeekCursorField.ExitWriteLock();
}
}
}
/// <summary>
/// Mark the dispatchers being preempted to stop
/// </summary>
/// <param name="coresToRelease">the cores to release</param>
/// <param name="plannedToShutdownTaskIds">the task ids planned to be shutdown</param>
private void MarkStoppingFlags(IEnumerable <string> plannedToShutdownTaskIds, IEnumerable <string> exitingTaskIds)
{
BrokerTracing.TraceVerbose(
"[GracefulPreemptionHandler].MarkStoppingFlags: stop the ExitIfPossible dispatchers, total count = {0}",
exitingTaskIds.Count());
// Occupied too much resource, stop the prefetch to release.
this.dispatcherManager.StopDispatchers(exitingTaskIds.Union(plannedToShutdownTaskIds), null);
}
/// <summary>
/// Async Pattern
/// Begin method for ProcessMessage
/// </summary>
/// <param name="request">request message</param>
/// <param name="callback">async callback</param>
/// <param name="asyncState">async state</param>
/// <returns>async result</returns>
public IAsyncResult BeginProcessMessage(Message request, AsyncCallback callback, object asyncState)
{
//
// if the connection is not ready (waitone timeout), let following BeginProcessMessage method throw exception.
//
BrokerTracing.TraceVerbose("[AzureServiceClient]. BeginProcessMessage: Send message at client {0}.", this);
return(this.ServiceClient.BeginProcessMessage(request, callback, asyncState));
}
/// <summary>
/// Invoke the cached callback for specified target queue name.
/// </summary>
/// <param name="messageIdMapping">
/// mapping from message Id to request/response name
/// </param>
/// <param name="queueName">
/// target request/response queue name
/// </param>
/// <param name="e">
/// exception occurred when access the queue
/// </param>
private void TriggerCallback(ConcurrentDictionary <UniqueId, string> messageIdMapping, string queueName, StorageException e)
{
BrokerTracing.TraceVerbose(
"[AzureQueueManager].TriggerCallback: Handle the invalid queue {0}",
queueName);
while (messageIdMapping.Values.Contains <string>(queueName, StringComparer.InvariantCultureIgnoreCase))
{
List <UniqueId> messageIds = new List <UniqueId>();
string tmpQueueName;
foreach (UniqueId id in messageIdMapping.Keys)
{
if (messageIdMapping.TryGetValue(id, out tmpQueueName))
{
if (string.Equals(queueName, tmpQueueName, StringComparison.InvariantCultureIgnoreCase))
{
messageIds.Add(id);
}
}
}
BrokerTracing.TraceVerbose(
"[AzureQueueManager].TriggerCallback: Invoke callback for {0} messages.",
messageIds.Count);
foreach (UniqueId messageId in messageIds)
{
QueueAsyncResult result;
if (this.callbacks.TryGetValue(messageId, out result))
{
BrokerTracing.TraceVerbose(
"[AzureQueueManager].TriggerCallback: Get callback for message {0} from callbacks.",
messageId);
// following method deletes callback and deletes message from mapping
this.CompleteCallback(result, null, e);
}
else
{
BrokerTracing.TraceWarning(
"[AzureQueueManager].TriggerCallback: Can not get callback for message {0} from callbacks.",
messageId);
}
if (!messageIdMapping.TryRemove(messageId, out tmpQueueName))
{
BrokerTracing.TraceWarning(
"[AzureQueueManager].TriggerCallback: Can not remove message {0} from mapping.",
messageId);
}
}
}
}
/// <summary>
/// Initializes a new instance of the AzureServiceClient class.
/// </summary>
/// <param name="binding">binding information</param>
/// <param name="remoteAddress">remote address</param>
public AzureServiceClient(Binding binding, EndpointAddress remoteAddress)
{
this.binding = binding;
this.remoteAddress = remoteAddress;
this.serviceClient = new ServiceClient(this.binding, this.remoteAddress);
BrokerTracing.TraceVerbose("[AzureServiceClient]. AzureServiceClient: Create a new client {0}", this.serviceClient.ToString());
this.serviceClient.ClientCredentials.ServiceCertificate.Authentication.CertificateValidationMode = X509CertificateValidationMode.None;
Utility.SetAzureClientCertificate(this.serviceClient.ChannelFactory.Credentials);
}
/// <summary>
/// Send out request to the target proxy or host.
/// </summary>
/// <param name="data">dispatch data</param>
/// <param name="dispatchId">dispatch Id</param>
/// <param name="clientIndex">client index</param>
public virtual void SendRequest(DispatchData data, Guid dispatchId, int clientIndex)
{
Message requestMessage = data.BrokerQueueItem.Message;
Guid messageId = Utility.GetMessageIdFromMessage(requestMessage);
// Bug 12045: Convert message before preparing it as some headers
// might be added during preparation.
// Check version
if (requestMessage.Headers.MessageVersion != this.BackendBinding.MessageVersion)
{
BrokerTracing.TraceVerbose("[RequestSender].SendRequest: Convert message version from {0} to {1}", requestMessage.Headers.MessageVersion, this.BackendBinding.MessageVersion);
requestMessage = Utility.ConvertMessage(requestMessage, this.BackendBinding.MessageVersion);
}
this.PrepareMessage(requestMessage, dispatchId, this.Dispatcher.PassBindingFlags[clientIndex]);
data.Client = this.IServiceClient;
// Bug #16197: reserve request message action in case it gets dropped when wcf processes the message
data.RequestAction = requestMessage.Headers.Action;
data.DispatchTime = DateTime.Now;
this.IServiceClient.BeginProcessMessage(
requestMessage,
this.ProcessMessageCallback,
data);
this.lastSentTime = DateTime.UtcNow;
// Notice: the request may complete fast, and it is disposed when response comes back before following code executes.
// So don't access item.Message in following code.
// Bug #13430:
// (1) Define "try" as request sent by broker, so any request sent by broker successfully should
// have "current try count + 1" and not retried any more if retrylimit=0.
// (2) Don't increase the try count if EndpointNotFoundException occurs.
// (3) In "Burst to Azure" mode, connections are shared by dispatchers. The connection is possible to
// be killed because of failure in one dispatcher. Don't increase the try count in such case.
// Increase the try count when BeginProcessMessage succeeds.
data.BrokerQueueItem.TryCount++;
BrokerTracing.TraceVerbose(
BrokerTracing.GenerateTraceString(
"RequestSender",
"SendRequest",
this.TaskId,
clientIndex,
this.IServiceClient.ToString(),
messageId,
string.Format("Sent out message and increase the try count to {0}", data.BrokerQueueItem.TryCount)));
}
/// <summary>
/// Increase the count
/// </summary>
/// <returns>Return false means the count is already 0 and needs to return immediately</returns>
public bool IncreaseCount()
{
if (this.refCount <= 0)
{
BrokerTracing.TraceVerbose("[ReferenceObject] Ref count is 0 when increasing, return false.");
return(false);
}
Interlocked.Increment(ref this.refCount);
return(true);
}