/// <summary> /// Attach to an existing session /// </summary> /// <param name="attachInfo">indicating the session attach information</param> /// <param name="durable">indicating a value whether a durable session is to be created</param> /// <param name="timeoutMilliseconds">indicating the timeout</param> /// <param name="binding">indicating the binding</param> /// <returns>returns session instance</returns> public override async Task <SessionBase> AttachSession(SessionAttachInfo attachInfo, bool durable, int timeoutMilliseconds, Binding binding) { Debug.Assert(attachInfo is SessionAttachInfo, "[OnPremiseSessionFactory].AttachSession: attachInfo's type must be SessionAttachInfo."); DateTime targetTimeout; Utility.ThrowIfNull(attachInfo, "attachInfo"); Utility.ThrowIfInvalidTimeout(timeoutMilliseconds, "timeoutMilliseconds"); if (timeoutMilliseconds == Timeout.Infinite) { targetTimeout = DateTime.MaxValue; } else { targetTimeout = DateTime.Now.AddMilliseconds(timeoutMilliseconds); } SessionBase.TraceSource.TraceEvent(TraceEventType.Information, 0, "[Session:{0}] Start to attach session...", attachInfo.SessionId); IResourceProvider resourceProvider = null; SessionInfo info = null; CredType typeOfExpectedCred = CredType.None; try { int retry = 0; bool askForCredential = false; // allow users to try credential at most MaxRetryCount times int askForCredentialTimes = 0; while (Utility.CanRetry(retry, askForCredential, askForCredentialTimes)) { retry++; try { SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Start to attach session.", attachInfo.SessionId); Stopwatch watch = new Stopwatch(); watch.Start(); if (attachInfo.UseAad) { // Authentication handled by AADUtil } else if (SoaHelper.IsSchedulerOnAzure(attachInfo.Headnode) || SoaHelper.IsSchedulerOnIaaS(attachInfo.Headnode) || (attachInfo.TransportScheme & TransportScheme.Http) == TransportScheme.Http) { askForCredential = HpcSessionCredUtil.RetrieveCredentialOnAzure(attachInfo); if (askForCredential) { askForCredentialTimes++; } } else { askForCredential = await HpcSessionCredUtil.RetrieveCredentialOnPremise(attachInfo, typeOfExpectedCred, binding).ConfigureAwait(false); if (askForCredential) { askForCredentialTimes++; } HpcSessionCredUtil.CheckCredential(attachInfo); HpcSessionCredUtil.CheckCredential(attachInfo); } resourceProvider = BuildResourceProvider(attachInfo, durable, binding); watch.Stop(); // re-calculate the timeout to exclude the timespan for getting credential try { targetTimeout = targetTimeout.AddMilliseconds(watch.ElapsedMilliseconds); } catch (ArgumentOutOfRangeException) { } info = await resourceProvider.GetResourceInfo(attachInfo, SessionBase.GetTimeout(targetTimeout)).ConfigureAwait(false); SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Successfully got resource info. BrokerLauncherEpr = {1}", attachInfo.SessionId, info.BrokerLauncherEpr); // If the session is an inprocess broker session, info.UseInprocessBroker will be set // to true by HpcSession service. Need to set attachInfo.UseInprocessBroker to true // in order to build correct broker factory instance. if (info.UseInprocessBroker) { attachInfo.UseInprocessBroker = true; } // If debug mode is enabled, need to set info.UseInprocessBroker to true in order to // build correct broker factory instance. if (attachInfo.DebugModeEnabled) { info.UseInprocessBroker = true; } if (SoaHelper.IsSchedulerOnIaaS(attachInfo.Headnode)) { string suffix = SoaHelper.GetSuffixFromHeadNodeEpr(attachInfo.Headnode); info.BrokerLauncherEpr = SoaHelper.UpdateEprWithCloudServiceName(info.BrokerLauncherEpr, suffix); if (info.BrokerEpr != null) { SoaHelper.UpdateEprWithCloudServiceName(info.BrokerEpr, suffix); } if (info.ControllerEpr != null) { SoaHelper.UpdateEprWithCloudServiceName(info.ControllerEpr, suffix); } if (info.ResponseEpr != null) { SoaHelper.UpdateEprWithCloudServiceName(info.ResponseEpr, suffix); } } HpcSessionCredUtil.SaveCrendential(attachInfo); break; } catch (SessionException) { throw; } catch (EndpointNotFoundException) { SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:{0}] EndpointNotFoundException occured while getting resource info.", attachInfo.SessionId); SessionBase.HandleEndpointNotFoundException(attachInfo.Headnode); } catch (AuthenticationException e) { SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:{0}] AuthenticationException occured while attaching session: {1}", attachInfo.SessionId, e); attachInfo.ClearCredential(); SessionBase.PurgeCredential(attachInfo); if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes)) { if (typeOfExpectedCred == CredType.None) { typeOfExpectedCred = await CredUtil.GetCredTypeFromClusterAsync(attachInfo, binding).ConfigureAwait(false); } if (resourceProvider is IDisposable) { ((IDisposable)resourceProvider).Dispose(); } continue; } else { throw; } } catch (MessageSecurityException e) { SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:{0}] MessageSecurityException occured while attaching session: {1}", attachInfo.SessionId, e); attachInfo.ClearCredential(); SessionBase.PurgeCredential(attachInfo); if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes)) { if (typeOfExpectedCred == CredType.None) { typeOfExpectedCred = await CredUtil.GetCredTypeFromClusterAsync(attachInfo, binding).ConfigureAwait(false); } if (resourceProvider is IDisposable) { ((IDisposable)resourceProvider).Dispose(); } continue; } else { throw; } } catch (FaultException <SessionFault> ex) { typeOfExpectedCred = Utility.CanRetry(retry, askForCredential, askForCredentialTimes) ? CredUtil.GetCredTypeFromFaultCode(ex.Detail.Code) : CredType.None; SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:{0}] Fault exception occured while allocating resource. FaultCode = {1}", attachInfo.SessionId, ex.Detail.Code); if (typeOfExpectedCred == CredType.None) { throw Utility.TranslateFaultException(ex); } else { attachInfo.ClearCredential(); if (resourceProvider is IDisposable) { ((IDisposable)resourceProvider).Dispose(); } continue; } } catch (CommunicationException e) { throw new SessionException(SOAFaultCode.ConnectSessionLauncherFailure, SR.ConnectSessionLauncherFailure, e); } catch (TimeoutException e) { throw new TimeoutException(string.Format(SR.ConnectSessionLauncherTimeout, Constant.DefaultCreateSessionTimeout), e); } catch (Exception e) { throw new SessionException(SOAFaultCode.UnknownError, e.ToString()); } } // while } finally { IDisposable disposableObject = resourceProvider as IDisposable; if (disposableObject != null) { disposableObject.Dispose(); } } if (String.IsNullOrEmpty(info.BrokerLauncherEpr) && !info.UseInprocessBroker) { if ((info.JobState & (JobState.Configuring | JobState.ExternalValidation | JobState.Queued | JobState.Running | JobState.Submitted | JobState.Validating)) != 0) { throw new SessionException(string.Format(SR.AttachConfiguringSession, attachInfo.SessionId)); } else { throw new SessionException(string.Format(SR.AttachNoBrokerSession, attachInfo.SessionId)); } } IBrokerFactory brokerFactory = BuildBrokerFactory(attachInfo, durable); try { return(await brokerFactory.AttachBroker(attachInfo, info, SessionBase.GetTimeout(targetTimeout), binding).ConfigureAwait(false)); } finally { IDisposable disposableObject = brokerFactory as IDisposable; if (disposableObject != null) { disposableObject.Dispose(); } } }
/// <summary> /// Create session /// </summary> /// <param name="startInfo">indicating session start information</param> /// <param name="durable">indicating a value whether a durable session is to be created</param> /// <param name="timeoutMilliseconds">indicating the timeout</param> /// <param name="binding">indicating the binding</param> /// <returns>returns session instance</returns> public override async Task <SessionBase> CreateSession(SessionStartInfo startInfo, bool durable, int timeoutMilliseconds, Binding binding) { SessionBase.CheckSessionStartInfo(startInfo); DateTime targetTimeout = DateTime.Now.AddMilliseconds(Constant.DefaultCreateSessionTimeout); // if following env variable is set, we will launch the service host in an admin job. string adminJobEnv = Environment.GetEnvironmentVariable(SessionBase.EnvVarNameForAdminJob, EnvironmentVariableTarget.Process); if (!string.IsNullOrEmpty(adminJobEnv) && adminJobEnv.Equals("1", StringComparison.InvariantCultureIgnoreCase)) { startInfo.AdminJobForHostInDiag = true; } SessionAllocateInfoContract sessionAllocateInfo = null; CredType typeOfExpectedCred = CredType.None; IResourceProvider resourceProvider = null; IList <string> newDataClients = null; try { int retry = 0; bool askForCredential = false; // allow users to try credential at most MaxRetryCount times int askForCredentialTimes = 0; while (Utility.CanRetry(retry, askForCredential, askForCredentialTimes)) { retry++; try { SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:Unknown] Start to create session for on-premise cluster. IsDurable = {0}, RetryCount = {1}, AskForCredentialTimes = {2}, TimeCost = {3}", durable, retry, askForCredentialTimes, targetTimeout); Stopwatch watch = new Stopwatch(); watch.Start(); if (startInfo.UseAad) { // Authentication handled by AADUtil } else if (SoaHelper.IsSchedulerOnAzure(startInfo.Headnode) || SoaHelper.IsSchedulerOnIaaS(startInfo.Headnode) || (startInfo.TransportScheme & TransportScheme.Http) == TransportScheme.Http || (startInfo.TransportScheme & TransportScheme.NetHttp) == TransportScheme.NetHttp) { askForCredential = HpcSessionCredUtil.RetrieveCredentialOnAzure(startInfo); if (askForCredential) { askForCredentialTimes++; } } else { askForCredential = await HpcSessionCredUtil.RetrieveCredentialOnPremise(startInfo, typeOfExpectedCred, binding).ConfigureAwait(false); if (askForCredential) { askForCredentialTimes++; } HpcSessionCredUtil.CheckCredential(startInfo); } resourceProvider = BuildResourceProvider(startInfo, durable, binding); if (((newDataClients == null) || (newDataClients.Count == 0)) && (startInfo.DependFiles != null) && (startInfo.DependFiles.Count > 0)) { // Upload the data files required for this session newDataClients = this.UploadAzureFiles(startInfo); } watch.Stop(); // re-calculate the timeout to exclude the timespan for getting credential targetTimeout = targetTimeout.AddMilliseconds(watch.ElapsedMilliseconds); sessionAllocateInfo = await resourceProvider.AllocateResource(startInfo, durable, SessionBase.GetTimeout(targetTimeout)).ConfigureAwait(false); SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Successfully allocated resource.", sessionAllocateInfo.Id); if (sessionAllocateInfo.BrokerLauncherEpr != null) { if (SoaHelper.IsSchedulerOnIaaS(startInfo.Headnode)) { string suffix = SoaHelper.GetSuffixFromHeadNodeEpr(startInfo.Headnode); for (int i = 0; i < sessionAllocateInfo.BrokerLauncherEpr.Length; i++) { sessionAllocateInfo.BrokerLauncherEpr[i] = SoaHelper.UpdateEprWithCloudServiceName(sessionAllocateInfo.BrokerLauncherEpr[i], suffix); } } HpcSessionCredUtil.SaveCrendential(startInfo, binding); SessionBase.TraceSource.TraceInformation("Get the EPR list from headnode. number of eprs={0}", sessionAllocateInfo.BrokerLauncherEpr.Length); break; } } catch (EndpointNotFoundException e) { this.CleanUpDataClients(startInfo, newDataClients); SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "[Session:Unknown] EndpointNotFoundException occured while allocating resource: {0}", e); SessionBase.HandleEndpointNotFoundException(startInfo.Headnode); } catch (AuthenticationException e) { SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] AuthenticationException occured while allocating resource: {0}", e); startInfo.ClearCredential(); SessionBase.PurgeCredential(startInfo); if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes)) { if (typeOfExpectedCred == CredType.None) { typeOfExpectedCred = await CredUtil.GetCredTypeFromClusterAsync(startInfo, binding).ConfigureAwait(false); } if (resourceProvider is IDisposable) { ((IDisposable)resourceProvider).Dispose(); } continue; } else { if (sessionAllocateInfo != null) { await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false); } throw; } } catch (MessageSecurityException e) { SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] MessageSecurityException occured while allocating resource: {0}", e); startInfo.ClearCredential(); SessionBase.PurgeCredential(startInfo); if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes)) { if (typeOfExpectedCred == CredType.None) { typeOfExpectedCred = await CredUtil.GetCredTypeFromClusterAsync(startInfo, binding).ConfigureAwait(false); } if (resourceProvider is IDisposable) { ((IDisposable)resourceProvider).Dispose(); } continue; } else { if (sessionAllocateInfo != null) { await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false); } throw; } } catch (FaultException <SessionFault> ex) { typeOfExpectedCred = Utility.CanRetry(retry, askForCredential, askForCredentialTimes) ? CredUtil.GetCredTypeFromFaultCode(ex.Detail.Code) : CredType.None; SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] Fault exception occured while allocating resource. FaultCode = {0}. Exception = {1}", ex.Detail.Code, ex.ToString()); if (typeOfExpectedCred == CredType.None) { this.CleanUpDataClients(startInfo, newDataClients); if (sessionAllocateInfo != null) { await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false); } throw Utility.TranslateFaultException(ex); } else { startInfo.ClearCredential(); if (resourceProvider is IDisposable) { ((IDisposable)resourceProvider).Dispose(); } continue; } } catch (CommunicationException e) { this.CleanUpDataClients(startInfo, newDataClients); if (sessionAllocateInfo != null) { await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false); } throw new SessionException(SOAFaultCode.ConnectSessionLauncherFailure, SR.ConnectSessionLauncherFailure, e); } catch (TimeoutException e) { SessionBase.TraceSource.TraceInformation(e.ToString()); this.CleanUpDataClients(startInfo, newDataClients); if (sessionAllocateInfo != null) { await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false); } throw new TimeoutException(string.Format(SR.ConnectSessionLauncherTimeout, Constant.DefaultCreateSessionTimeout), e); } catch (Exception e) { SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] Exception occured while allocating resource: {0}", e); this.CleanUpDataClients(startInfo, newDataClients); if (sessionAllocateInfo != null) { await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false); } throw; } if (sessionAllocateInfo.BrokerLauncherEpr == null && startInfo.UseSessionPool) { // If the session launcher picks a session from the pool, it returns the seesion id. // eprs is null at this case. Try to attach to the session. try { if (sessionAllocateInfo.SessionInfo == null) { SessionBase.TraceSource.TraceInformation("[Session:{0}] Attempt to attach to session {0} which is part of the session pool.", sessionAllocateInfo.Id); return(await AttachSession(new SessionAttachInfo(startInfo.Headnode, sessionAllocateInfo.Id), durable, timeoutMilliseconds, binding).ConfigureAwait(false)); } else { SessionBase.TraceSource.TraceInformation("[Session:{0}] Attempt to attach to broker of the session {0} which is part of the session pool.", sessionAllocateInfo.Id); return(await AttachBroker(startInfo, sessionAllocateInfo.SessionInfo, durable, timeoutMilliseconds, binding).ConfigureAwait(false)); } } catch (Exception e) { SessionBase.TraceSource.TraceEvent(TraceEventType.Error, 0, "Failed to attach to the session {0}. {1}", sessionAllocateInfo.Id, e); if (Utility.CanRetry(retry, askForCredential, askForCredentialTimes)) { if (resourceProvider is IDisposable) { ((IDisposable)resourceProvider).Dispose(); } continue; } else { throw; } } } } IBrokerFactory brokerFactory = BuildBrokerFactory(startInfo, durable); try { return(await brokerFactory.CreateBroker(startInfo, sessionAllocateInfo.Id, targetTimeout, sessionAllocateInfo.BrokerLauncherEpr, binding).ConfigureAwait(false)); } catch { // Free resource if failed to create broker or create session this.CleanUpDataClients(startInfo, newDataClients); await resourceProvider.FreeResource(startInfo, sessionAllocateInfo.Id).ConfigureAwait(false); throw; } finally { if (brokerFactory is IDisposable) { ((IDisposable)brokerFactory).Dispose(); } } } finally { if (resourceProvider != null) { if (resourceProvider is IDisposable) { ((IDisposable)resourceProvider).Dispose(); } } } }
/// <summary> /// The callback function for the allocate operation to session launcher /// </summary> /// <param name="ar">the async result</param> private void AllocateCallback(IAsyncResult ar) { SessionLauncherClient client = (SessionLauncherClient)ar.AsyncState; SessionInfoContract sessionInfo; try { try { bool durable = (GetType() == typeof(DurableSessionAsyncResult)); if (durable) { this.eprCanidates = client.EndAllocateDurable(out this.sessionid, out serviceVersion, out sessionInfo, ar); } else { this.eprCanidates = client.EndAllocate(out this.sessionid, out serviceVersion, out sessionInfo, ar); } SessionBase.TraceSource.TraceEvent(TraceEventType.Verbose, 0, "[Session:{0}] Successfully allocated resource. ServiceVersion = {1}", this.sessionid, serviceVersion); this.eprPtr = 0; // If HpcSession returns a version, pass it to HpcBroker if (!String.IsNullOrEmpty(serviceVersion)) { try { this.StartInfo.Data.ServiceVersion = new Version(serviceVersion); } catch { throw new SessionException(SR.InvalidServiceVersionReturned); } } SessionBase.SaveCrendential(this.StartInfo, this.binding); if (this.eprCanidates == null || this.eprCanidates.Length == 0) { throw new SessionException(SR.NoBrokerNodeFound); } if (SoaHelper.IsSchedulerOnIaaS(this.StartInfo.Headnode)) { string suffix = SoaHelper.GetSuffixFromHeadNodeEpr(this.StartInfo.Headnode); for (int i = 0; i < this.eprCanidates.Length; i++) { this.eprCanidates[i] = SoaHelper.UpdateEprWithCloudServiceName(this.eprCanidates[i], suffix); } } SessionBase.TraceSource.TraceInformation("Get the EPR list from headnode. number of eprCanidates={0}", this.eprCanidates.Length); if (!this.Canceled) { this.ConnectToNextBroker(); } else { // Bug 11765: If the operation is canceled, when allocating // resource, do cleanup the resource this.CleanupOnFailure().GetAwaiter().GetResult(); } } catch (AuthenticationException ex) { SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] AuthenticationException occured while allocating resource: {0}", ex); this.StartInfo.ClearCredential(); SessionBase.PurgeCredential(this.StartInfo); if (retry > 0) { //TODO: remove sync wait BeginAllocate(this.StartInfo, CredType.None).GetAwaiter().GetResult(); } else { this.MarkFinish(new AuthenticationException(SR.AuthenticationFailed, ex)); } } catch (MessageSecurityException ex) { SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] MessageSecurityException occured while allocating resource: {0}", ex); this.StartInfo.ClearCredential(); SessionBase.PurgeCredential(this.StartInfo); if (retry > 0) { //TODO: remove sync wait BeginAllocate(this.StartInfo, CredType.None).GetAwaiter().GetResult(); } else { this.MarkFinish(new AuthenticationException(SR.AuthenticationFailed, ex)); } } catch (FaultException <SessionFault> ex) { int faultCode = (ex as FaultException <SessionFault>).Detail.Code; CredType type = CredUtil.GetCredTypeFromFaultCode(faultCode); SessionBase.TraceSource.TraceEvent(TraceEventType.Warning, 0, "[Session:Unknown] Fault exception occured while allocating resource. FaultCode = {0}, CredType = {1}", faultCode, type); if (type != CredType.None) { this.StartInfo.ClearCredential(); if (retry > 0) { //TODO: remove sync wait BeginAllocate(this.StartInfo, type).GetAwaiter().GetResult(); } else { this.MarkFinish(new AuthenticationException(SR.AuthenticationFailed, ex)); } } else { throw; } } } catch (FaultException <SessionFault> ex) { this.MarkFinish(Utility.TranslateFaultException(ex)); } catch (Exception ex) { this.MarkFinish(ex); } finally { Utility.SafeCloseCommunicateObject(client); } }