/// <exception cref="System.IO.IOException"/> public virtual void TransitionToStandby(HAServiceProtocol.StateChangeRequestInfo reqInfo) { lock (this) { // call refreshAdminAcls before HA state transition // for the case that adminAcls have been updated in previous active RM try { RefreshAdminAcls(false); } catch (YarnException ex) { throw new ServiceFailedException("Can not execute refreshAdminAcls", ex); } UserGroupInformation user = CheckAccess("transitionToStandby"); CheckHaStateChange(reqInfo); try { rm.TransitionToStandby(true); RMAuditLogger.LogSuccess(user.GetShortUserName(), "transitionToStandby", "RMHAProtocolService" ); } catch (Exception e) { RMAuditLogger.LogFailure(user.GetShortUserName(), "transitionToStandby", string.Empty , "RMHAProtocolService", "Exception transitioning to standby"); throw new ServiceFailedException("Error when transitioning to Standby mode", e); } } }
/// <summary> /// Utility method to verify if the current user has access based on the /// passed /// <see cref="Org.Apache.Hadoop.Security.Authorize.AccessControlList"/> /// </summary> /// <param name="authorizer"> /// the /// <see cref="Org.Apache.Hadoop.Security.Authorize.AccessControlList"/> /// to check against /// </param> /// <param name="method">the method name to be logged</param> /// <param name="module">like AdminService or NodeLabelManager</param> /// <param name="Log">the logger to use</param> /// <returns> /// /// <see cref="Org.Apache.Hadoop.Security.UserGroupInformation"/> /// of the current user /// </returns> /// <exception cref="System.IO.IOException"/> public static UserGroupInformation VerifyAdminAccess(YarnAuthorizationProvider authorizer , string method, string module, Log Log) { UserGroupInformation user; try { user = UserGroupInformation.GetCurrentUser(); } catch (IOException ioe) { Log.Warn("Couldn't get current user", ioe); RMAuditLogger.LogFailure("UNKNOWN", method, string.Empty, "AdminService", "Couldn't get current user" ); throw; } if (!authorizer.IsAdmin(user)) { Log.Warn("User " + user.GetShortUserName() + " doesn't have permission" + " to call '" + method + "'"); RMAuditLogger.LogFailure(user.GetShortUserName(), method, string.Empty, module, RMAuditLogger.AuditConstants .UnauthorizedUser); throw new AccessControlException("User " + user.GetShortUserName() + " doesn't have permission" + " to call '" + method + "'"); } if (Log.IsTraceEnabled()) { Log.Trace(method + " invoked by user " + user.GetShortUserName()); } return(user); }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> private YarnException LogAndWrapException(IOException ioe, string user, string argName , string msg) { Log.Info("Exception " + msg, ioe); RMAuditLogger.LogFailure(user, argName, string.Empty, "AdminService", "Exception " + msg); return(RPCUtil.GetRemoteException(ioe)); }
/// <exception cref="Org.Apache.Hadoop.Ipc.StandbyException"/> private void CheckRMStatus(string user, string argName, string msg) { if (!IsRMActive()) { RMAuditLogger.LogFailure(user, argName, string.Empty, "AdminService", "ResourceManager is not active. Can not " + msg); ThrowStandbyException(); } }
protected internal virtual void WriteAuditLog(ApplicationId appId) { RMApp app = rmContext.GetRMApps()[appId]; string operation = "UNKONWN"; bool success = false; switch (app.GetState()) { case RMAppState.Failed: { operation = RMAuditLogger.AuditConstants.FinishFailedApp; break; } case RMAppState.Finished: { operation = RMAuditLogger.AuditConstants.FinishSuccessApp; success = true; break; } case RMAppState.Killed: { operation = RMAuditLogger.AuditConstants.FinishKilledApp; success = true; break; } default: { break; } } if (success) { RMAuditLogger.LogSuccess(app.GetUser(), operation, "RMAppManager", app.GetApplicationId ()); } else { StringBuilder diag = app.GetDiagnostics(); string msg = diag == null ? null : diag.ToString(); RMAuditLogger.LogFailure(app.GetUser(), operation, msg, "RMAppManager", "App failed with state: " + app.GetState(), appId); } }
/// <exception cref="System.IO.IOException"/> public virtual void TransitionToActive(HAServiceProtocol.StateChangeRequestInfo reqInfo ) { lock (this) { // call refreshAdminAcls before HA state transition // for the case that adminAcls have been updated in previous active RM try { RefreshAdminAcls(false); } catch (YarnException ex) { throw new ServiceFailedException("Can not execute refreshAdminAcls", ex); } UserGroupInformation user = CheckAccess("transitionToActive"); CheckHaStateChange(reqInfo); try { rm.TransitionToActive(); } catch (Exception e) { RMAuditLogger.LogFailure(user.GetShortUserName(), "transitionToActive", string.Empty , "RMHAProtocolService", "Exception transitioning to active"); throw new ServiceFailedException("Error when transitioning to Active mode", e); } try { // call all refresh*s for active RM to get the updated configurations. RefreshAll(); } catch (Exception e) { Log.Error("RefreshAll failed so firing fatal event", e); rmContext.GetDispatcher().GetEventHandler().Handle(new RMFatalEvent(RMFatalEventType .TransitionToActiveFailed, e)); throw new ServiceFailedException("Error on refreshAll during transistion to Active" , e); } RMAuditLogger.LogSuccess(user.GetShortUserName(), "transitionToActive", "RMHAProtocolService" ); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual FinishApplicationMasterResponse FinishApplicationMaster(FinishApplicationMasterRequest request) { ApplicationAttemptId applicationAttemptId = AuthorizeRequest().GetApplicationAttemptId (); ApplicationId appId = applicationAttemptId.GetApplicationId(); RMApp rmApp = rmContext.GetRMApps()[applicationAttemptId.GetApplicationId()]; // checking whether the app exits in RMStateStore at first not to throw // ApplicationDoesNotExistInCacheException before and after // RM work-preserving restart. if (rmApp.IsAppFinalStateStored()) { Log.Info(rmApp.GetApplicationId() + " unregistered successfully. "); return(FinishApplicationMasterResponse.NewInstance(true)); } ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId ]; if (Lock == null) { ThrowApplicationDoesNotExistInCacheException(applicationAttemptId); } // Allow only one thread in AM to do finishApp at a time. lock (Lock) { if (!HasApplicationMasterRegistered(applicationAttemptId)) { string message = "Application Master is trying to unregister before registering for: " + appId; Log.Error(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appId].GetUser(), RMAuditLogger.AuditConstants .UnregisterAm, string.Empty, "ApplicationMasterService", message, appId, applicationAttemptId ); throw new ApplicationMasterNotRegisteredException(message); } this.amLivelinessMonitor.ReceivedPing(applicationAttemptId); rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptUnregistrationEvent (applicationAttemptId, request.GetTrackingUrl(), request.GetFinalApplicationStatus (), request.GetDiagnostics())); // For UnmanagedAMs, return true so they don't retry return(FinishApplicationMasterResponse.NewInstance(rmApp.GetApplicationSubmissionContext ().GetUnmanagedAM())); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest(); ApplicationAttemptId appAttemptId = amrmTokenIdentifier.GetApplicationAttemptId(); ApplicationId applicationId = appAttemptId.GetApplicationId(); this.amLivelinessMonitor.ReceivedPing(appAttemptId); /* check if its in cache */ ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId]; if (Lock == null) { string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache."; Log.Error(message); throw new ApplicationAttemptNotFoundException(message); } lock (Lock) { AllocateResponse lastResponse = Lock.GetAllocateResponse(); if (!HasApplicationMasterRegistered(appAttemptId)) { string message = "AM is not registered for known application attempt: " + appAttemptId + " or RM had restarted after AM registered . AM should re-register."; Log.Info(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService" , message, applicationId, appAttemptId); throw new ApplicationMasterNotRegisteredException(message); } if ((request.GetResponseId() + 1) == lastResponse.GetResponseId()) { /* old heartbeat */ return(lastResponse); } else { if (request.GetResponseId() + 1 < lastResponse.GetResponseId()) { string message = "Invalid responseId in AllocateRequest from application attempt: " + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() + 1); throw new InvalidApplicationMasterRequestException(message); } } //filter illegal progress values float filteredProgress = request.GetProgress(); if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity || filteredProgress < 0) { request.SetProgress(0); } else { if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity) { request.SetProgress(1); } } // Send the status update to the appAttempt. this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent (appAttemptId, request.GetProgress())); IList <ResourceRequest> ask = request.GetAskList(); IList <ContainerId> release = request.GetReleaseList(); ResourceBlacklistRequest blacklistRequest = request.GetResourceBlacklistRequest(); IList <string> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest. GetBlacklistAdditions() : Sharpen.Collections.EmptyList; IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals () : Sharpen.Collections.EmptyList; RMApp app = this.rmContext.GetRMApps()[applicationId]; // set label expression for Resource Requests if resourceName=ANY ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext(); foreach (ResourceRequest req in ask) { if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName ())) { req.SetNodeLabelExpression(asc.GetNodeLabelExpression()); } } // sanity check try { RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability (), app.GetQueue(), rScheduler, rmContext); } catch (InvalidResourceRequestException e) { Log.Warn("Invalid resource ask by application " + appAttemptId, e); throw; } try { RMServerUtils.ValidateBlacklistRequest(blacklistRequest); } catch (InvalidResourceBlacklistRequestException e) { Log.Warn("Invalid blacklist request by application " + appAttemptId, e); throw; } // In the case of work-preserving AM restart, it's possible for the // AM to release containers from the earlier attempt. if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts ()) { try { RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId); } catch (InvalidContainerReleaseException e) { Log.Warn("Invalid container release by application " + appAttemptId, e); throw; } } // Send new requests to appAttempt. Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions , blacklistRemovals); if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty()) { Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals); } RMAppAttempt appAttempt = app.GetRMAppAttempt(appAttemptId); AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse >(); if (!allocation.GetContainers().IsEmpty()) { allocateResponse.SetNMTokens(allocation.GetNMTokens()); } // update the response with the deltas of node status changes IList <RMNode> updatedNodes = new AList <RMNode>(); if (app.PullRMNodeUpdates(updatedNodes) > 0) { IList <NodeReport> updatedNodeReports = new AList <NodeReport>(); foreach (RMNode rmNode in updatedNodes) { SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID ()); Resource used = BuilderUtils.NewResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.GetUsedResource(); numContainers = schedulerNodeReport.GetNumContainers(); } NodeId nodeId = rmNode.GetNodeID(); NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode. GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels ()); updatedNodeReports.AddItem(report); } allocateResponse.SetUpdatedNodes(updatedNodeReports); } allocateResponse.SetAllocatedContainers(allocation.GetContainers()); allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers ()); allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1); allocateResponse.SetAvailableResources(allocation.GetResourceLimit()); allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes()); // add preemption to the allocateResponse message (if any) allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation)); // update AMRMToken if the token is rolled-up MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData (); if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier .GetKeyId()) { RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt; Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt .GetAMRMToken(); if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId()) { Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: " + applicationId); amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId ); appAttemptImpl.SetAMRMToken(amrmToken); } allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword (), amrmToken.GetService().ToString())); } /* * As we are updating the response inside the lock object so we don't * need to worry about unregister call occurring in between (which * removes the lock object). */ Lock.SetAllocateResponse(allocateResponse); return(allocateResponse); } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual RegisterApplicationMasterResponse RegisterApplicationMaster(RegisterApplicationMasterRequest request) { AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest(); ApplicationAttemptId applicationAttemptId = amrmTokenIdentifier.GetApplicationAttemptId (); ApplicationId appID = applicationAttemptId.GetApplicationId(); ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId ]; if (Lock == null) { RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants .RegisterAm, "Application doesn't exist in cache " + applicationAttemptId, "ApplicationMasterService" , "Error in registering application master", appID, applicationAttemptId); ThrowApplicationDoesNotExistInCacheException(applicationAttemptId); } // Allow only one thread in AM to do registerApp at a time. lock (Lock) { AllocateResponse lastResponse = Lock.GetAllocateResponse(); if (HasApplicationMasterRegistered(applicationAttemptId)) { string message = "Application Master is already registered : " + appID; Log.Warn(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appID].GetUser(), RMAuditLogger.AuditConstants .RegisterAm, string.Empty, "ApplicationMasterService", message, appID, applicationAttemptId ); throw new InvalidApplicationMasterRequestException(message); } this.amLivelinessMonitor.ReceivedPing(applicationAttemptId); RMApp app = this.rmContext.GetRMApps()[appID]; // Setting the response id to 0 to identify if the // application master is register for the respective attemptid lastResponse.SetResponseId(0); Lock.SetAllocateResponse(lastResponse); Log.Info("AM registration " + applicationAttemptId); this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptRegistrationEvent (applicationAttemptId, request.GetHost(), request.GetRpcPort(), request.GetTrackingUrl ())); RMAuditLogger.LogSuccess(app.GetUser(), RMAuditLogger.AuditConstants.RegisterAm, "ApplicationMasterService", appID, applicationAttemptId); // Pick up min/max resource from scheduler... RegisterApplicationMasterResponse response = recordFactory.NewRecordInstance <RegisterApplicationMasterResponse >(); response.SetMaximumResourceCapability(rScheduler.GetMaximumResourceCapability(app .GetQueue())); response.SetApplicationACLs(app.GetRMAppAttempt(applicationAttemptId).GetSubmissionContext ().GetAMContainerSpec().GetApplicationACLs()); response.SetQueue(app.GetQueue()); if (UserGroupInformation.IsSecurityEnabled()) { Log.Info("Setting client token master key"); response.SetClientToAMTokenMasterKey(ByteBuffer.Wrap(rmContext.GetClientToAMTokenSecretManager ().GetMasterKey(applicationAttemptId).GetEncoded())); } // For work-preserving AM restart, retrieve previous attempts' containers // and corresponding NM tokens. if (app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts ()) { IList <Container> transferredContainers = ((AbstractYarnScheduler)rScheduler).GetTransferredContainers (applicationAttemptId); if (!transferredContainers.IsEmpty()) { response.SetContainersFromPreviousAttempts(transferredContainers); IList <NMToken> nmTokens = new AList <NMToken>(); foreach (Container container in transferredContainers) { try { NMToken token = rmContext.GetNMTokenSecretManager().CreateAndGetNMToken(app.GetUser (), applicationAttemptId, container); if (null != token) { nmTokens.AddItem(token); } } catch (ArgumentException e) { // if it's a DNS issue, throw UnknowHostException directly and // that // will be automatically retried by RMProxy in RPC layer. if (e.InnerException is UnknownHostException) { throw (UnknownHostException)e.InnerException; } } } response.SetNMTokensFromPreviousAttempts(nmTokens); Log.Info("Application " + appID + " retrieved " + transferredContainers.Count + " containers from previous" + " attempts and " + nmTokens.Count + " NM tokens."); } } response.SetSchedulerResourceTypes(rScheduler.GetSchedulingResourceTypes()); return(response); } }