private YarnServiceProtos.NMTokenProto ConvertToProtoFormat(NMToken token) { lock (this) { return(((NMTokenPBImpl)token).GetProto()); } }
public virtual NMToken CreateAndGetNMToken(string applicationSubmitter, ApplicationAttemptId appAttemptId, Container container) { try { this.readLock.Lock(); HashSet <NodeId> nodeSet = this.appAttemptToNodeKeyMap[appAttemptId]; NMToken nmToken = null; if (nodeSet != null) { if (!nodeSet.Contains(container.GetNodeId())) { Log.Info("Sending NMToken for nodeId : " + container.GetNodeId() + " for container : " + container.GetId()); Token token = CreateNMToken(container.GetId().GetApplicationAttemptId(), container .GetNodeId(), applicationSubmitter); nmToken = NMToken.NewInstance(container.GetNodeId(), token); nodeSet.AddItem(container.GetNodeId()); } } return(nmToken); } finally { this.readLock.Unlock(); } }
// Create container token and NMToken altogether, if either of them fails for // some reason like DNS unavailable, do not return this container and keep it // in the newlyAllocatedContainers waiting to be refetched. public virtual SchedulerApplicationAttempt.ContainersAndNMTokensAllocation PullNewlyAllocatedContainersAndNMTokens () { lock (this) { IList <Container> returnContainerList = new AList <Container>(newlyAllocatedContainers .Count); IList <NMToken> nmTokens = new AList <NMToken>(); for (IEnumerator <RMContainer> i = newlyAllocatedContainers.GetEnumerator(); i.HasNext ();) { RMContainer rmContainer = i.Next(); Container container = rmContainer.GetContainer(); try { // create container token and NMToken altogether. container.SetContainerToken(rmContext.GetContainerTokenSecretManager().CreateContainerToken (container.GetId(), container.GetNodeId(), GetUser(), container.GetResource(), container .GetPriority(), rmContainer.GetCreationTime(), this.logAggregationContext)); NMToken nmToken = rmContext.GetNMTokenSecretManager().CreateAndGetNMToken(GetUser (), GetApplicationAttemptId(), container); if (nmToken != null) { nmTokens.AddItem(nmToken); } } catch (ArgumentException e) { // DNS might be down, skip returning this container. Log.Error("Error trying to assign container token and NM token to" + " an allocated container " + container.GetId(), e); continue; } returnContainerList.AddItem(container); i.Remove(); rmContainer.Handle(new RMContainerEvent(rmContainer.GetContainerId(), RMContainerEventType .Acquired)); } return(new SchedulerApplicationAttempt.ContainersAndNMTokensAllocation(returnContainerList , nmTokens)); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual RegisterApplicationMasterResponse RegisterApplicationMaster(RegisterApplicationMasterRequest request) { AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest(); ApplicationAttemptId applicationAttemptId = amrmTokenIdentifier.GetApplicationAttemptId (); ApplicationId appID = applicationAttemptId.GetApplicationId(); ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId ]; if (Lock == null) { RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants .RegisterAm, "Application doesn't exist in cache " + applicationAttemptId, "ApplicationMasterService" , "Error in registering application master", appID, applicationAttemptId); ThrowApplicationDoesNotExistInCacheException(applicationAttemptId); } // Allow only one thread in AM to do registerApp at a time. lock (Lock) { AllocateResponse lastResponse = Lock.GetAllocateResponse(); if (HasApplicationMasterRegistered(applicationAttemptId)) { string message = "Application Master is already registered : " + appID; Log.Warn(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants .RegisterAm, string.Empty, "ApplicationMasterService", message, appID, applicationAttemptId ); throw new InvalidApplicationMasterRequestException(message); } this.amLivelinessMonitor.ReceivedPing(applicationAttemptId); RMApp app = this.rmContext.GetRMApps()[appID]; // Setting the response id to 0 to identify if the // application master is register for the respective attemptid lastResponse.SetResponseId(0); Lock.SetAllocateResponse(lastResponse); Log.Info("AM registration " + applicationAttemptId); this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptRegistrationEvent (applicationAttemptId, request.GetHost(), request.GetRpcPort(), request.GetTrackingUrl ())); RMAuditLogger.LogSuccess(app.GetUser(), RMAuditLogger.AuditConstants.RegisterAm, "ApplicationMasterService", appID, applicationAttemptId); // Pick up min/max resource from scheduler... RegisterApplicationMasterResponse response = recordFactory.NewRecordInstance <RegisterApplicationMasterResponse >(); response.SetMaximumResourceCapability(rScheduler.GetMaximumResourceCapability(app .GetQueue())); response.SetApplicationACLs(app.GetRMAppAttempt(applicationAttemptId).GetSubmissionContext ().GetAMContainerSpec().GetApplicationACLs()); response.SetQueue(app.GetQueue()); if (UserGroupInformation.IsSecurityEnabled()) { Log.Info("Setting client token master key"); response.SetClientToAMTokenMasterKey(ByteBuffer.Wrap(rmContext.GetClientToAMTokenSecretManager ().GetMasterKey(applicationAttemptId).GetEncoded())); } // For work-preserving AM restart, retrieve previous attempts' containers // and corresponding NM tokens. if (app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts ()) { IList <Container> transferredContainers = ((AbstractYarnScheduler)rScheduler).GetTransferredContainers (applicationAttemptId); if (!transferredContainers.IsEmpty()) { response.SetContainersFromPreviousAttempts(transferredContainers); IList <NMToken> nmTokens = new AList <NMToken>(); foreach (Container container in transferredContainers) { try { NMToken token = rmContext.GetNMTokenSecretManager().CreateAndGetNMToken(app.GetUser (), applicationAttemptId, container); if (null != token) { nmTokens.AddItem(token); } } catch (ArgumentException e) { // if it's a DNS issue, throw UnknowHostException directly and // that // will be automatically retried by RMProxy in RPC layer. if (e.InnerException is UnknownHostException) { throw (UnknownHostException)e.InnerException; } } } response.SetNMTokensFromPreviousAttempts(nmTokens); Log.Info("Application " + appID + " retrieved " + transferredContainers.Count + " containers from previous" + " attempts and " + nmTokens.Count + " NM tokens."); } } response.SetSchedulerResourceTypes(rScheduler.GetSchedulingResourceTypes()); return(response); } }