Beispiel #1
0
        /// <summary>
        /// Resolve session launcher node on IaaS
        /// </summary>
        /// <param name="context"></param>
        /// <param name="headnode"></param>
        /// <returns></returns>
        public static async Task <string> ResolveSessionLauncherNodeOnIaasAsync(this ITelepathyContext context, string headnode)
        {
            string sessionLauncher = await context.ResolveSessionLauncherNodeAsync().ConfigureAwait(false);

            if (SoaHelper.IsSchedulerOnIaaS(headnode))
            {
                string suffix = SoaHelper.GetSuffixFromHeadNodeEpr(headnode);
                sessionLauncher += suffix;
            }

            return(sessionLauncher);
        }
        /// <summary>
        /// Create session
        /// </summary>
        /// <param name="startInfo">indicating session start information</param>
        /// <param name="durable">indicating a value whether a durable session is to be created</param>
        /// <param name="timeoutMilliseconds">indicating the timeout</param>
        /// <param name="binding">indicating the binding</param>
        /// <returns>returns session instance</returns>
        public override async Task <SessionBase> CreateSession(SessionStartInfo startInfo, bool durable, int timeoutMilliseconds, Binding binding)
        {
            SessionBase.CheckSessionStartInfo(startInfo);
            DateTime targetTimeout = DateTime.Now.AddMilliseconds(Constant.DefaultCreateSessionTimeout);

            // if following env variable is set, we will launch the service host in an admin job.
            string adminJobEnv = Environment.GetEnvironmentVariable(SessionBase.EnvVarNameForAdminJob, EnvironmentVariableTarget.Process);

            if (!string.IsNullOrEmpty(adminJobEnv) && adminJobEnv.Equals("1", StringComparison.InvariantCultureIgnoreCase))
            {
                startInfo.AdminJobForHostInDiag = true;
            }

            SessionAllocateInfoContract sessionAllocateInfo = null;

            CredType          typeOfExpectedCred = CredType.None;
            IResourceProvider resourceProvider   = null;
            IList <string>    newDataClients     = null;

            try
            {
                int retry = 0;

                bool askForCredential = false;

                // allow users to try credential at most MaxRetryCount times
                int askForCredentialTimes = 0;

                while (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
                {
                    retry++;

                    try
                    {
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:Unknown] Start to create session for on-premise cluster. IsDurable = {0}, RetryCount = {1}, AskForCredentialTimes = {2}, TimeCost = {3}", durable, retry, askForCredentialTimes, targetTimeout);

                        Stopwatch watch = new Stopwatch();
                        watch.Start();

                        if (startInfo.UseAad)
                        {
                            // Authentication handled by AADUtil
                        }
                        else if (SoaHelper.IsSchedulerOnAzure(startInfo.Headnode) || SoaHelper.IsSchedulerOnIaaS(startInfo.Headnode) || (startInfo.TransportScheme & TransportScheme.Http) == TransportScheme.Http || (startInfo.TransportScheme & TransportScheme.NetHttp) == TransportScheme.NetHttp)
                        {
                            askForCredential = HpcSessionCredUtil.RetrieveCredentialOnAzure(startInfo);

                            if (askForCredential)
                            {
                                askForCredentialTimes++;
                            }
                        }
                        else
                        {
                            askForCredential = await HpcSessionCredUtil.RetrieveCredentialOnPremise(startInfo, typeOfExpectedCred, binding).ConfigureAwait(false);

                            if (askForCredential)
                            {
                                askForCredentialTimes++;
                            }

                            HpcSessionCredUtil.CheckCredential(startInfo);
                        }

                        resourceProvider = BuildResourceProvider(startInfo, durable, binding);
                        if (((newDataClients == null) || (newDataClients.Count == 0)) &&
                            (startInfo.DependFiles != null) && (startInfo.DependFiles.Count > 0))
                        {
                            // Upload the data files required for this session
                            newDataClients = this.UploadAzureFiles(startInfo);
                        }

                        watch.Stop();

                        // re-calculate the timeout to exclude the timespan for getting credential
                        targetTimeout       = targetTimeout.AddMilliseconds(watch.ElapsedMilliseconds);
                        sessionAllocateInfo = await resourceProvider.AllocateResource(startInfo, durable, SessionBase.GetTimeout(targetTimeout)).ConfigureAwait(false);

                        SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Successfully allocated resource.", sessionAllocateInfo.Id);

                        if (sessionAllocateInfo.BrokerLauncherEpr != null)
                        {
                            if (SoaHelper.IsSchedulerOnIaaS(startInfo.Headnode))
                            {
                                string suffix = SoaHelper.GetSuffixFromHeadNodeEpr(startInfo.Headnode);
                                for (int i = 0; i < sessionAllocateInfo.BrokerLauncherEpr.Length; i++)
                                {
                                    sessionAllocateInfo.BrokerLauncherEpr[i] = SoaHelper.UpdateEprWithCloudServiceName(sessionAllocateInfo.BrokerLauncherEpr[i], suffix);
                                }
                            }
                            HpcSessionCredUtil.SaveCrendential(startInfo, binding);
                            SessionBase.TraceSource.TraceInformation("Get the EPR list from headnode. number of eprs={0}", sessionAllocateInfo.BrokerLauncherEpr.Length);
                            break;
                        }
                    }
                    catch (EndpointNotFoundException e)
                    {
                        this.CleanUpDataClients(startInfo, newDataClients);
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:Unknown] EndpointNotFoundException occured while allocating resource: {0}", e);
                        SessionBase.HandleEndpointNotFoundException(startInfo.Headnode);
                    }
                    catch (AuthenticationException e)
                    {
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] AuthenticationException occured while allocating resource: {0}", e);

                        startInfo.ClearCredential();
                        SessionBase.PurgeCredential(startInfo);
                        if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
                        {
                            if (typeOfExpectedCred == CredType.None)
                            {
                                typeOfExpectedCred = await CredUtil.GetCredTypeFromClusterAsync(startInfo, binding).ConfigureAwait(false);
                            }

                            if (resourceProvider is IDisposable)
                            {
                                ((IDisposable)resourceProvider).Dispose();
                            }
                            continue;
                        }
                        else
                        {
                            if (sessionAllocateInfo != null)
                            {
                                await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
                            }
                            throw;
                        }
                    }
                    catch (MessageSecurityException e)
                    {
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] MessageSecurityException occured while allocating resource: {0}", e);

                        startInfo.ClearCredential();
                        SessionBase.PurgeCredential(startInfo);
                        if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
                        {
                            if (typeOfExpectedCred == CredType.None)
                            {
                                typeOfExpectedCred = await CredUtil.GetCredTypeFromClusterAsync(startInfo, binding).ConfigureAwait(false);
                            }

                            if (resourceProvider is IDisposable)
                            {
                                ((IDisposable)resourceProvider).Dispose();
                            }
                            continue;
                        }
                        else
                        {
                            if (sessionAllocateInfo != null)
                            {
                                await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
                            }
                            throw;
                        }
                    }
                    catch (FaultException <SessionFault> ex)
                    {
                        typeOfExpectedCred = Utility.CanRetry(retry, askForCredential, askForCredentialTimes) ?
                                             CredUtil.GetCredTypeFromFaultCode(ex.Detail.Code) : CredType.None;
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] Fault exception occured while allocating resource. FaultCode = {0}. Exception = {1}", ex.Detail.Code, ex.ToString());

                        if (typeOfExpectedCred == CredType.None)
                        {
                            this.CleanUpDataClients(startInfo, newDataClients);
                            if (sessionAllocateInfo != null)
                            {
                                await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
                            }
                            throw Utility.TranslateFaultException(ex);
                        }
                        else
                        {
                            startInfo.ClearCredential();

                            if (resourceProvider is IDisposable)
                            {
                                ((IDisposable)resourceProvider).Dispose();
                            }
                            continue;
                        }
                    }
                    catch (CommunicationException e)
                    {
                        this.CleanUpDataClients(startInfo, newDataClients);
                        if (sessionAllocateInfo != null)
                        {
                            await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
                        }
                        throw new SessionException(SOAFaultCode.ConnectSessionLauncherFailure, SR.ConnectSessionLauncherFailure, e);
                    }
                    catch (TimeoutException e)
                    {
                        SessionBase.TraceSource.TraceInformation(e.ToString());
                        this.CleanUpDataClients(startInfo, newDataClients);
                        if (sessionAllocateInfo != null)
                        {
                            await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
                        }
                        throw new TimeoutException(string.Format(SR.ConnectSessionLauncherTimeout, Constant.DefaultCreateSessionTimeout), e);
                    }
                    catch (Exception e)
                    {
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] Exception occured while allocating resource: {0}", e);

                        this.CleanUpDataClients(startInfo, newDataClients);
                        if (sessionAllocateInfo != null)
                        {
                            await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);
                        }
                        throw;
                    }

                    if (sessionAllocateInfo.BrokerLauncherEpr == null && startInfo.UseSessionPool)
                    {
                        // If the session launcher picks a session from the pool, it returns the seesion id.
                        // eprs is null at this case. Try to attach to the session.

                        try
                        {
                            if (sessionAllocateInfo.SessionInfo == null)
                            {
                                SessionBase.TraceSource.TraceInformation("[Session:{0}] Attempt to attach to session {0} which is part of the session pool.", sessionAllocateInfo.Id);
                                return(await AttachSession(new SessionAttachInfo(startInfo.Headnode, sessionAllocateInfo.Id), durable, timeoutMilliseconds, binding).ConfigureAwait(false));
                            }
                            else
                            {
                                SessionBase.TraceSource.TraceInformation("[Session:{0}] Attempt to attach to broker of the session {0} which is part of the session pool.", sessionAllocateInfo.Id);
                                return(await AttachBroker(startInfo, sessionAllocateInfo.SessionInfo, durable, timeoutMilliseconds, binding).ConfigureAwait(false));
                            }
                        }
                        catch (Exception e)
                        {
                            SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "Failed to attach to the session {0}. {1}", sessionAllocateInfo.Id, e);
                            if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
                            {
                                if (resourceProvider is IDisposable)
                                {
                                    ((IDisposable)resourceProvider).Dispose();
                                }
                                continue;
                            }
                            else
                            {
                                throw;
                            }
                        }
                    }
                }

                IBrokerFactory brokerFactory = BuildBrokerFactory(startInfo, durable);

                try
                {
                    return(await brokerFactory.CreateBroker(startInfo, sessionAllocateInfo.Id, targetTimeout, sessionAllocateInfo.BrokerLauncherEpr, binding).ConfigureAwait(false));
                }
                catch
                {
                    // Free resource if failed to create broker or create session
                    this.CleanUpDataClients(startInfo, newDataClients);
                    await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false);

                    throw;
                }
                finally
                {
                    if (brokerFactory is IDisposable)
                    {
                        ((IDisposable)brokerFactory).Dispose();
                    }
                }
            }
            finally
            {
                if (resourceProvider != null)
                {
                    if (resourceProvider is IDisposable)
                    {
                        ((IDisposable)resourceProvider).Dispose();
                    }
                }
            }
        }
Beispiel #3
0
        /// <summary>
        /// Attach to a broker, returns session instance
        /// </summary>
        /// <param name="attachInfo">indicating the attach information</param>
        /// <param name="info">indicating the session info to be updated</param>
        /// <param name="timeout">indicating the timeout</param>
        /// <param name="binding">indicting the binding</param>
        /// <returns>returns the session instance</returns>
        public Task <SessionBase> AttachBroker(SessionAttachInfo attachInfo, SessionInfo info, TimeSpan timeout, Binding binding)
        {
            SessionBase.TraceSource.TraceInformation("[Session:{0}] Try to attach broker...", attachInfo.SessionId);
            BrokerLauncherClient broker = new BrokerLauncherClient(new Uri(info.BrokerLauncherEpr), attachInfo, binding);

            broker.InnerChannel.OperationTimeout = timeout;
            try
            {
                BrokerInitializationResult result = broker.Attach(info.Id);
                info.BrokerEpr                       = result.BrokerEpr;
                info.ControllerEpr                   = result.ControllerEpr;
                info.ResponseEpr                     = result.ResponseEpr;
                info.ServiceOperationTimeout         = result.ServiceOperationTimeout;
                info.MaxMessageSize                  = result.MaxMessageSize;
                info.ClientBrokerHeartbeatInterval   = result.ClientBrokerHeartbeatInterval;
                info.ClientBrokerHeartbeatRetryCount = result.ClientBrokerHeartbeatRetryCount;
                info.BrokerUniqueId                  = result.BrokerUniqueId;

                info.UseAzureQueue         = result.UseAzureQueue;
                info.AzureRequestQueueUris = result.AzureRequestQueueUris;
                info.AzureRequestBlobUri   = result.AzureRequestBlobUri;

                info.Username         = attachInfo.Username;
                info.InternalPassword = attachInfo.InternalPassword;
                info.Headnode         = attachInfo.Headnode;

                info.UseWindowsClientCredential = attachInfo.UseWindowsClientCredential;
            }
            catch (FaultException <SessionFault> e)
            {
                SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:{0}] Fault exception occured while attaching broker: {1}. FaultCode = {2}", attachInfo.SessionId, e, e.Detail.Code);
                throw Utility.TranslateFaultException(e);
            }
            catch (CommunicationException e)
            {
                SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:{0}] Failed to attach to broker: {1}", attachInfo.SessionId, e);
                throw new SessionException(SOAFaultCode.ConnectBrokerLauncherFailure, SR.ConnectBrokerLauncherFailure, e);
            }
            catch (Exception e)
            {
                SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:{0}] Failed to attach to broker: {1}", attachInfo.SessionId, e);
                throw new SessionException(SOAFaultCode.UnknownError, e.ToString());
            }
            finally
            {
                Utility.SafeCloseCommunicateObject(broker);
            }

            if (SoaHelper.IsSchedulerOnIaaS(attachInfo.Headnode))
            {
                string suffix = SoaHelper.GetSuffixFromHeadNodeEpr(attachInfo.Headnode);
                if (info.BrokerEpr != null)
                {
                    SoaHelper.UpdateEprWithCloudServiceName(info.BrokerEpr, suffix);
                }

                if (info.ControllerEpr != null)
                {
                    SoaHelper.UpdateEprWithCloudServiceName(info.ControllerEpr, suffix);
                }

                if (info.ResponseEpr != null)
                {
                    SoaHelper.UpdateEprWithCloudServiceName(info.ResponseEpr, suffix);
                }
            }

            if (this.durable)
            {
                if (!info.Durable)
                {
                    throw new SessionException(SOAFaultCode.InvalidAttachInteractiveSession, SR.InvalidAttachInteractiveSession);
                }
#if net40
                return(TaskEx.FromResult <SessionBase>(new DurableSession(info, attachInfo.Headnode, binding)));
#else
                return(Task.FromResult <SessionBase>(new DurableSession(info, attachInfo.Headnode, binding)));
#endif
            }
            else
            {
                if (info.Durable)
                {
                    throw new SessionException(SOAFaultCode.InvalidAttachDurableSession, SR.InvalidAttachDurableSession);
                }
#if net40
                return(TaskEx.FromResult <SessionBase>(new V3Session(info, attachInfo.Headnode, true, binding)));
#else
                return(Task.FromResult <SessionBase>(new V3Session(info, attachInfo.Headnode, true, binding)));
#endif
            }
        }
        /// <summary>
        /// Attach to an existing session
        /// </summary>
        /// <param name="attachInfo">indicating the session attach information</param>
        /// <param name="durable">indicating a value whether a durable session is to be created</param>
        /// <param name="timeoutMilliseconds">indicating the timeout</param>
        /// <param name="binding">indicating the binding</param>
        /// <returns>returns session instance</returns>
        public override async Task <SessionBase> AttachSession(SessionAttachInfo attachInfo, bool durable, int timeoutMilliseconds, Binding binding)
        {
            Debug.Assert(attachInfo is SessionAttachInfo, "[OnPremiseSessionFactory].AttachSession: attachInfo's type must be SessionAttachInfo.");
            DateTime targetTimeout;

            Utility.ThrowIfNull(attachInfo, "attachInfo");
            Utility.ThrowIfInvalidTimeout(timeoutMilliseconds, "timeoutMilliseconds");

            if (timeoutMilliseconds == Timeout.Infinite)
            {
                targetTimeout = DateTime.MaxValue;
            }
            else
            {
                targetTimeout = DateTime.Now.AddMilliseconds(timeoutMilliseconds);
            }

            SessionBase.TraceSource.TraceEvent(TraceEventType.Information, 0, "[Session:{0}] Start to attach session...", attachInfo.SessionId);

            IResourceProvider resourceProvider   = null;
            SessionInfo       info               = null;
            CredType          typeOfExpectedCred = CredType.None;

            try
            {
                int  retry            = 0;
                bool askForCredential = false;

                // allow users to try credential at most MaxRetryCount times
                int askForCredentialTimes = 0;
                while (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
                {
                    retry++;

                    try
                    {
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Start to attach session.", attachInfo.SessionId);
                        Stopwatch watch = new Stopwatch();
                        watch.Start();

                        if (attachInfo.UseAad)
                        {
                            // Authentication handled by AADUtil
                        }
                        else if (SoaHelper.IsSchedulerOnAzure(attachInfo.Headnode) || SoaHelper.IsSchedulerOnIaaS(attachInfo.Headnode) || (attachInfo.TransportScheme & TransportScheme.Http) == TransportScheme.Http)
                        {
                            askForCredential = HpcSessionCredUtil.RetrieveCredentialOnAzure(attachInfo);

                            if (askForCredential)
                            {
                                askForCredentialTimes++;
                            }
                        }
                        else
                        {
                            askForCredential = await HpcSessionCredUtil.RetrieveCredentialOnPremise(attachInfo, typeOfExpectedCred, binding).ConfigureAwait(false);

                            if (askForCredential)
                            {
                                askForCredentialTimes++;
                            }

                            HpcSessionCredUtil.CheckCredential(attachInfo);
                            HpcSessionCredUtil.CheckCredential(attachInfo);
                        }

                        resourceProvider = BuildResourceProvider(attachInfo, durable, binding);
                        watch.Stop();

                        // re-calculate the timeout to exclude the timespan for getting credential
                        try
                        {
                            targetTimeout = targetTimeout.AddMilliseconds(watch.ElapsedMilliseconds);
                        }
                        catch (ArgumentOutOfRangeException)
                        {
                        }

                        info = await resourceProvider.GetResourceInfo(attachInfo, SessionBase.GetTimeout(targetTimeout)).ConfigureAwait(false);

                        SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Successfully got resource info. BrokerLauncherEpr = {1}", attachInfo.SessionId, info.BrokerLauncherEpr);

                        // If the session is an inprocess broker session, info.UseInprocessBroker will be set
                        // to true by HpcSession service. Need to set attachInfo.UseInprocessBroker to true
                        // in order to build correct broker factory instance.
                        if (info.UseInprocessBroker)
                        {
                            attachInfo.UseInprocessBroker = true;
                        }

                        // If debug mode is enabled, need to set info.UseInprocessBroker to true in order to
                        // build correct broker factory instance.
                        if (attachInfo.DebugModeEnabled)
                        {
                            info.UseInprocessBroker = true;
                        }

                        if (SoaHelper.IsSchedulerOnIaaS(attachInfo.Headnode))
                        {
                            string suffix = SoaHelper.GetSuffixFromHeadNodeEpr(attachInfo.Headnode);
                            info.BrokerLauncherEpr = SoaHelper.UpdateEprWithCloudServiceName(info.BrokerLauncherEpr, suffix);
                            if (info.BrokerEpr != null)
                            {
                                SoaHelper.UpdateEprWithCloudServiceName(info.BrokerEpr, suffix);
                            }

                            if (info.ControllerEpr != null)
                            {
                                SoaHelper.UpdateEprWithCloudServiceName(info.ControllerEpr, suffix);
                            }

                            if (info.ResponseEpr != null)
                            {
                                SoaHelper.UpdateEprWithCloudServiceName(info.ResponseEpr, suffix);
                            }
                        }
                        HpcSessionCredUtil.SaveCrendential(attachInfo);
                        break;
                    }
                    catch (SessionException)
                    {
                        throw;
                    }
                    catch (EndpointNotFoundException)
                    {
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:{0}] EndpointNotFoundException occured while getting resource info.", attachInfo.SessionId);
                        SessionBase.HandleEndpointNotFoundException(attachInfo.Headnode);
                    }
                    catch (AuthenticationException e)
                    {
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:{0}] AuthenticationException occured while attaching session: {1}", attachInfo.SessionId, e);

                        attachInfo.ClearCredential();
                        SessionBase.PurgeCredential(attachInfo);
                        if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
                        {
                            if (typeOfExpectedCred == CredType.None)
                            {
                                typeOfExpectedCred = await CredUtil.GetCredTypeFromClusterAsync(attachInfo, binding).ConfigureAwait(false);
                            }

                            if (resourceProvider is IDisposable)
                            {
                                ((IDisposable)resourceProvider).Dispose();
                            }
                            continue;
                        }
                        else
                        {
                            throw;
                        }
                    }
                    catch (MessageSecurityException e)
                    {
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:{0}] MessageSecurityException occured while attaching session: {1}", attachInfo.SessionId, e);

                        attachInfo.ClearCredential();
                        SessionBase.PurgeCredential(attachInfo);
                        if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes))
                        {
                            if (typeOfExpectedCred == CredType.None)
                            {
                                typeOfExpectedCred = await CredUtil.GetCredTypeFromClusterAsync(attachInfo, binding).ConfigureAwait(false);
                            }

                            if (resourceProvider is IDisposable)
                            {
                                ((IDisposable)resourceProvider).Dispose();
                            }
                            continue;
                        }
                        else
                        {
                            throw;
                        }
                    }
                    catch (FaultException <SessionFault> ex)
                    {
                        typeOfExpectedCred = Utility.CanRetry(retry, askForCredential, askForCredentialTimes) ?
                                             CredUtil.GetCredTypeFromFaultCode(ex.Detail.Code) : CredType.None;
                        SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:{0}] Fault exception occured while allocating resource. FaultCode = {1}", attachInfo.SessionId, ex.Detail.Code);

                        if (typeOfExpectedCred == CredType.None)
                        {
                            throw Utility.TranslateFaultException(ex);
                        }
                        else
                        {
                            attachInfo.ClearCredential();

                            if (resourceProvider is IDisposable)
                            {
                                ((IDisposable)resourceProvider).Dispose();
                            }
                            continue;
                        }
                    }
                    catch (CommunicationException e)
                    {
                        throw new SessionException(SOAFaultCode.ConnectSessionLauncherFailure, SR.ConnectSessionLauncherFailure, e);
                    }
                    catch (TimeoutException e)
                    {
                        throw new TimeoutException(string.Format(SR.ConnectSessionLauncherTimeout, Constant.DefaultCreateSessionTimeout), e);
                    }
                    catch (Exception e)
                    {
                        throw new SessionException(SOAFaultCode.UnknownError, e.ToString());
                    }
                } // while
            }
            finally
            {
                IDisposable disposableObject = resourceProvider as IDisposable;
                if (disposableObject != null)
                {
                    disposableObject.Dispose();
                }
            }

            if (String.IsNullOrEmpty(info.BrokerLauncherEpr) && !info.UseInprocessBroker)
            {
                if ((info.JobState &
                     (JobState.Configuring
                      | JobState.ExternalValidation
                      | JobState.Queued
                      | JobState.Running
                      | JobState.Submitted
                      | JobState.Validating)) != 0)
                {
                    throw new SessionException(string.Format(SR.AttachConfiguringSession, attachInfo.SessionId));
                }
                else
                {
                    throw new SessionException(string.Format(SR.AttachNoBrokerSession, attachInfo.SessionId));
                }
            }

            IBrokerFactory brokerFactory = BuildBrokerFactory(attachInfo, durable);

            try
            {
                return(await brokerFactory.AttachBroker(attachInfo, info, SessionBase.GetTimeout(targetTimeout), binding).ConfigureAwait(false));
            }
            finally
            {
                IDisposable disposableObject = brokerFactory as IDisposable;
                if (disposableObject != null)
                {
                    disposableObject.Dispose();
                }
            }
        }
        /// <summary>
        /// The callback function for the allocate operation to session launcher
        /// </summary>
        /// <param name="ar">the async result</param>
        private void AllocateCallback(IAsyncResult ar)
        {
            SessionLauncherClient client = (SessionLauncherClient)ar.AsyncState;
            SessionInfoContract   sessionInfo;

            try
            {
                try
                {
                    bool durable = (GetType() == typeof(DurableSessionAsyncResult));

                    if (durable)
                    {
                        this.eprCanidates = client.EndAllocateDurable(out this.sessionid, out serviceVersion, out sessionInfo, ar);
                    }
                    else
                    {
                        this.eprCanidates = client.EndAllocate(out this.sessionid, out serviceVersion, out sessionInfo, ar);
                    }

                    SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Successfully allocated resource. ServiceVersion = {1}", this.sessionid, serviceVersion);
                    this.eprPtr = 0;

                    // If HpcSession returns a version, pass it to HpcBroker
                    if (!String.IsNullOrEmpty(serviceVersion))
                    {
                        try
                        {
                            this.StartInfo.Data.ServiceVersion = new Version(serviceVersion);
                        }

                        catch
                        {
                            throw new SessionException(SR.InvalidServiceVersionReturned);
                        }
                    }

                    SessionBase.SaveCrendential(this.StartInfo, this.binding);

                    if (this.eprCanidates == null || this.eprCanidates.Length == 0)
                    {
                        throw new SessionException(SR.NoBrokerNodeFound);
                    }

                    if (SoaHelper.IsSchedulerOnIaaS(this.StartInfo.Headnode))
                    {
                        string suffix = SoaHelper.GetSuffixFromHeadNodeEpr(this.StartInfo.Headnode);
                        for (int i = 0; i < this.eprCanidates.Length; i++)
                        {
                            this.eprCanidates[i] = SoaHelper.UpdateEprWithCloudServiceName(this.eprCanidates[i], suffix);
                        }
                    }

                    SessionBase.TraceSource.TraceInformation("Get the EPR list from headnode. number of eprCanidates={0}", this.eprCanidates.Length);



                    if (!this.Canceled)
                    {
                        this.ConnectToNextBroker();
                    }
                    else
                    {
                        // Bug 11765: If the operation is canceled, when allocating
                        // resource, do cleanup the resource
                        this.CleanupOnFailure().GetAwaiter().GetResult();
                    }
                }
                catch (AuthenticationException ex)
                {
                    SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] AuthenticationException occured while allocating resource: {0}", ex);

                    this.StartInfo.ClearCredential();
                    SessionBase.PurgeCredential(this.StartInfo);

                    if (retry > 0)
                    {
                        //TODO: remove sync wait
                        BeginAllocate(this.StartInfo, CredType.None).GetAwaiter().GetResult();
                    }
                    else
                    {
                        this.MarkFinish(new AuthenticationException(SR.AuthenticationFailed, ex));
                    }
                }
                catch (MessageSecurityException ex)
                {
                    SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] MessageSecurityException occured while allocating resource: {0}", ex);

                    this.StartInfo.ClearCredential();
                    SessionBase.PurgeCredential(this.StartInfo);

                    if (retry > 0)
                    {
                        //TODO: remove sync wait
                        BeginAllocate(this.StartInfo, CredType.None).GetAwaiter().GetResult();
                    }
                    else
                    {
                        this.MarkFinish(new AuthenticationException(SR.AuthenticationFailed, ex));
                    }
                }
                catch (FaultException <SessionFault> ex)
                {
                    int      faultCode = (ex as FaultException <SessionFault>).Detail.Code;
                    CredType type      = CredUtil.GetCredTypeFromFaultCode(faultCode);
                    SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] Fault exception occured while allocating resource. FaultCode = {0}, CredType = {1}", faultCode, type);

                    if (type != CredType.None)
                    {
                        this.StartInfo.ClearCredential();

                        if (retry > 0)
                        {
                            //TODO: remove sync wait
                            BeginAllocate(this.StartInfo, type).GetAwaiter().GetResult();
                        }
                        else
                        {
                            this.MarkFinish(new AuthenticationException(SR.AuthenticationFailed, ex));
                        }
                    }
                    else
                    {
                        throw;
                    }
                }
            }
            catch (FaultException <SessionFault> ex)
            {
                this.MarkFinish(Utility.TranslateFaultException(ex));
            }
            catch (Exception ex)
            {
                this.MarkFinish(ex);
            }
            finally
            {
                Utility.SafeCloseCommunicateObject(client);
            }
        }