public virtual void RollMasterKey() { base.writeLock.Lock(); try { Log.Info("Rolling master-key for container-tokens"); if (this.currentMasterKey == null) { // Setting up for the first time. this.currentMasterKey = CreateNewMasterKey(); } else { this.nextMasterKey = CreateNewMasterKey(); Log.Info("Going to activate master-key with key-id " + this.nextMasterKey.GetMasterKey ().GetKeyId() + " in " + this.activationDelay + "ms"); this.timer.Schedule(new RMContainerTokenSecretManager.NextKeyActivator(this), this .activationDelay); } } finally { base.writeLock.Unlock(); } }
/// <exception cref="System.Exception"/> public virtual void TestAMRMTokenSecretManagerStateStore(RMStateStoreTestBase.RMStateStoreHelper stateStoreHelper) { System.Console.Out.WriteLine("Start testing"); RMStateStore store = stateStoreHelper.GetRMStateStore(); RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher (); store.SetRMDispatcher(dispatcher); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store); Configuration conf = new YarnConfiguration(); AMRMTokenSecretManager appTokenMgr = new AMRMTokenSecretManager(conf, rmContext); //create and save the first masterkey MasterKeyData firstMasterKeyData = appTokenMgr.CreateNewMasterKey(); AMRMTokenSecretManagerState state1 = AMRMTokenSecretManagerState.NewInstance(firstMasterKeyData .GetMasterKey(), null); rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state1, false); // load state store = stateStoreHelper.GetRMStateStore(); Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store); store.SetRMDispatcher(dispatcher); RMStateStore.RMState state = store.LoadState(); NUnit.Framework.Assert.IsNotNull(state.GetAMRMTokenSecretManagerState()); NUnit.Framework.Assert.AreEqual(firstMasterKeyData.GetMasterKey(), state.GetAMRMTokenSecretManagerState ().GetCurrentMasterKey()); NUnit.Framework.Assert.IsNull(state.GetAMRMTokenSecretManagerState().GetNextMasterKey ()); //create and save the second masterkey MasterKeyData secondMasterKeyData = appTokenMgr.CreateNewMasterKey(); AMRMTokenSecretManagerState state2 = AMRMTokenSecretManagerState.NewInstance(firstMasterKeyData .GetMasterKey(), secondMasterKeyData.GetMasterKey()); rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state2, true); // load state store = stateStoreHelper.GetRMStateStore(); Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store); store.SetRMDispatcher(dispatcher); RMStateStore.RMState state_2 = store.LoadState(); NUnit.Framework.Assert.IsNotNull(state_2.GetAMRMTokenSecretManagerState()); NUnit.Framework.Assert.AreEqual(firstMasterKeyData.GetMasterKey(), state_2.GetAMRMTokenSecretManagerState ().GetCurrentMasterKey()); NUnit.Framework.Assert.AreEqual(secondMasterKeyData.GetMasterKey(), state_2.GetAMRMTokenSecretManagerState ().GetNextMasterKey()); // re-create the masterKeyData based on the recovered masterkey // should have the same secretKey appTokenMgr.Recover(state_2); NUnit.Framework.Assert.AreEqual(appTokenMgr.GetCurrnetMasterKeyData().GetSecretKey (), firstMasterKeyData.GetSecretKey()); NUnit.Framework.Assert.AreEqual(appTokenMgr.GetNextMasterKeyData().GetSecretKey() , secondMasterKeyData.GetSecretKey()); store.Close(); }
private void UpdateAppAttemptKey(ApplicationAttemptId attempt, MasterKeyData key) { this.oldMasterKeys[attempt] = key; try { stateStore.StoreNMTokenApplicationMasterKey(attempt, key.GetMasterKey()); } catch (IOException e) { Log.Error("Unable to store master key for application " + attempt, e); } }
private void UpdateCurrentMasterKey(MasterKeyData key) { base.currentMasterKey = key; try { stateStore.StoreContainerTokenCurrentMasterKey(key.GetMasterKey()); } catch (IOException e) { Log.Error("Unable to update current master key in state store", e); } }
public virtual void Start() { if (this.currentMasterKey == null) { this.currentMasterKey = CreateNewMasterKey(); AMRMTokenSecretManagerState state = AMRMTokenSecretManagerState.NewInstance(this. currentMasterKey.GetMasterKey(), null); rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state, false); } this.timer.ScheduleAtFixedRate(new AMRMTokenSecretManager.MasterKeyRoller(this), rollingInterval, rollingInterval); }
private void UpdatePreviousMasterKey(MasterKeyData key) { previousMasterKey = key; try { stateStore.StoreContainerTokenPreviousMasterKey(key.GetMasterKey()); } catch (IOException e) { Log.Error("Unable to update previous master key in state store", e); } }
public virtual void ActivateNextMasterKey() { base.writeLock.Lock(); try { Log.Info("Activating next master key with id: " + this.nextMasterKey.GetMasterKey ().GetKeyId()); this.currentMasterKey = this.nextMasterKey; this.nextMasterKey = null; } finally { base.writeLock.Unlock(); } }
public virtual void ActivateNextMasterKey() { this.writeLock.Lock(); try { Log.Info("Activating next master key with id: " + this.nextMasterKey.GetMasterKey ().GetKeyId()); this.currentMasterKey = this.nextMasterKey; this.nextMasterKey = null; AMRMTokenSecretManagerState state = AMRMTokenSecretManagerState.NewInstance(this. currentMasterKey.GetMasterKey(), null); rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state, true); } finally { this.writeLock.Unlock(); } }
internal virtual void RollMasterKey() { this.writeLock.Lock(); try { Log.Info("Rolling master-key for amrm-tokens"); this.nextMasterKey = CreateNewMasterKey(); AMRMTokenSecretManagerState state = AMRMTokenSecretManagerState.NewInstance(this. currentMasterKey.GetMasterKey(), this.nextMasterKey.GetMasterKey()); rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state, true); this.timer.Schedule(new AMRMTokenSecretManager.NextKeyActivator(this), this.activationDelay ); } finally { this.writeLock.Unlock(); } }
/// <summary> /// This method will be used to verify NMTokens generated by different master /// keys. /// </summary> /// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/> public override byte[] RetrievePassword(NMTokenIdentifier identifier) { lock (this) { int keyId = identifier.GetKeyId(); ApplicationAttemptId appAttemptId = identifier.GetApplicationAttemptId(); /* * MasterKey used for retrieving password will be as follows. 1) By default * older saved master key will be used. 2) If identifier's master key id * matches that of previous master key id then previous key will be used. 3) * If identifier's master key id matches that of current master key id then * current key will be used. */ MasterKeyData oldMasterKey = oldMasterKeys[appAttemptId]; MasterKeyData masterKeyToUse = oldMasterKey; if (previousMasterKey != null && keyId == previousMasterKey.GetMasterKey().GetKeyId ()) { masterKeyToUse = previousMasterKey; } else { if (keyId == currentMasterKey.GetMasterKey().GetKeyId()) { masterKeyToUse = currentMasterKey; } } if (nodeId != null && !identifier.GetNodeId().Equals(nodeId)) { throw new SecretManager.InvalidToken("Given NMToken for application : " + appAttemptId .ToString() + " is not valid for current node manager." + "expected : " + nodeId .ToString() + " found : " + identifier.GetNodeId().ToString()); } if (masterKeyToUse != null) { byte[] password = RetrivePasswordInternal(identifier, masterKeyToUse); Log.Debug("NMToken password retrieved successfully!!"); return(password); } throw new SecretManager.InvalidToken("Given NMToken for application : " + appAttemptId .ToString() + " seems to have been generated illegally."); } }
/// <exception cref="System.IO.IOException"/> public virtual void Recover() { lock (this) { NMStateStoreService.RecoveredNMTokensState state = stateStore.LoadNMTokensState(); MasterKey key = state.GetCurrentMasterKey(); if (key != null) { base.currentMasterKey = new MasterKeyData(key, CreateSecretKey(((byte[])key.GetBytes ().Array()))); } key = state.GetPreviousMasterKey(); if (key != null) { previousMasterKey = new MasterKeyData(key, CreateSecretKey(((byte[])key.GetBytes( ).Array()))); } // restore the serial number from the current master key if (base.currentMasterKey != null) { base.serialNo = base.currentMasterKey.GetMasterKey().GetKeyId() + 1; } foreach (KeyValuePair <ApplicationAttemptId, MasterKey> entry in state.GetApplicationMasterKeys ()) { key = entry.Value; oldMasterKeys[entry.Key] = new MasterKeyData(key, CreateSecretKey(((byte[])key.GetBytes ().Array()))); } // reconstruct app to app attempts map appToAppAttemptMap.Clear(); foreach (ApplicationAttemptId attempt in oldMasterKeys.Keys) { ApplicationId app = attempt.GetApplicationId(); IList <ApplicationAttemptId> attempts = appToAppAttemptMap[app]; if (attempts == null) { attempts = new AList <ApplicationAttemptId>(); appToAppAttemptMap[app] = attempts; } attempts.AddItem(attempt); } } }
/// <summary>This will be called by startContainer.</summary> /// <remarks> /// This will be called by startContainer. It will add the master key into /// the cache used for starting this container. This should be called before /// validating the startContainer request. /// </remarks> /// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/> public virtual void AppAttemptStartContainer(NMTokenIdentifier identifier) { lock (this) { ApplicationAttemptId appAttemptId = identifier.GetApplicationAttemptId(); if (!appToAppAttemptMap.Contains(appAttemptId.GetApplicationId())) { // First application attempt for the given application appToAppAttemptMap[appAttemptId.GetApplicationId()] = new AList <ApplicationAttemptId >(); } MasterKeyData oldKey = oldMasterKeys[appAttemptId]; if (oldKey == null) { // This is a new application attempt. appToAppAttemptMap[appAttemptId.GetApplicationId()].AddItem(appAttemptId); } if (oldKey == null || oldKey.GetMasterKey().GetKeyId() != identifier.GetKeyId()) { // Update key only if it is modified. Log.Debug("NMToken key updated for application attempt : " + identifier.GetApplicationAttemptId ().ToString()); if (identifier.GetKeyId() == currentMasterKey.GetMasterKey().GetKeyId()) { UpdateAppAttemptKey(appAttemptId, currentMasterKey); } else { if (previousMasterKey != null && identifier.GetKeyId() == previousMasterKey.GetMasterKey ().GetKeyId()) { UpdateAppAttemptKey(appAttemptId, previousMasterKey); } else { throw new SecretManager.InvalidToken("Older NMToken should not be used while starting the container." ); } } } } }
public virtual void Recover(RMStateStore.RMState state) { if (state.GetAMRMTokenSecretManagerState() != null) { // recover the current master key MasterKey currentKey = state.GetAMRMTokenSecretManagerState().GetCurrentMasterKey (); this.currentMasterKey = new MasterKeyData(currentKey, CreateSecretKey(((byte[])currentKey .GetBytes().Array()))); // recover the next master key if not null MasterKey nextKey = state.GetAMRMTokenSecretManagerState().GetNextMasterKey(); if (nextKey != null) { this.nextMasterKey = new MasterKeyData(nextKey, CreateSecretKey(((byte[])nextKey. GetBytes().Array()))); this.timer.Schedule(new AMRMTokenSecretManager.NextKeyActivator(this), this.activationDelay ); } } }
/// <exception cref="System.IO.IOException"/> public virtual void Recover() { lock (this) { NMStateStoreService.RecoveredContainerTokensState state = stateStore.LoadContainerTokensState (); MasterKey key = state.GetCurrentMasterKey(); if (key != null) { base.currentMasterKey = new MasterKeyData(key, CreateSecretKey(((byte[])key.GetBytes ().Array()))); } key = state.GetPreviousMasterKey(); if (key != null) { previousMasterKey = new MasterKeyData(key, CreateSecretKey(((byte[])key.GetBytes( ).Array()))); } // restore the serial number from the current master key if (base.currentMasterKey != null) { base.serialNo = base.currentMasterKey.GetMasterKey().GetKeyId() + 1; } foreach (KeyValuePair <ContainerId, long> entry in state.GetActiveTokens()) { ContainerId containerId = entry.Key; long expTime = entry.Value; IList <ContainerId> containerList = recentlyStartedContainerTracker[expTime]; if (containerList == null) { containerList = new AList <ContainerId>(); recentlyStartedContainerTracker[expTime] = containerList; } if (!containerList.Contains(containerId)) { containerList.AddItem(containerId); } } } }
/// <summary> /// Override of this is to validate ContainerTokens generated by using /// different /// <see cref="Org.Apache.Hadoop.Yarn.Server.Api.Records.MasterKey"/> /// s. /// </summary> /// <exception cref="Org.Apache.Hadoop.Security.Token.SecretManager.InvalidToken"/> public override byte[] RetrievePassword(ContainerTokenIdentifier identifier) { lock (this) { int keyId = identifier.GetMasterKeyId(); MasterKeyData masterKeyToUse = null; if (this.previousMasterKey != null && keyId == this.previousMasterKey.GetMasterKey ().GetKeyId()) { // A container-launch has come in with a token generated off the last // master-key masterKeyToUse = this.previousMasterKey; } else { if (keyId == base.currentMasterKey.GetMasterKey().GetKeyId()) { // A container-launch has come in with a token generated off the current // master-key masterKeyToUse = base.currentMasterKey; } } if (nodeHostAddr != null && !identifier.GetNmHostAddress().Equals(nodeHostAddr)) { // Valid container token used for incorrect node. throw new SecretManager.InvalidToken("Given Container " + identifier.GetContainerID ().ToString() + " identifier is not valid for current Node manager. Expected : " + nodeHostAddr + " Found : " + identifier.GetNmHostAddress()); } if (masterKeyToUse != null) { return(RetrievePasswordInternal(identifier, masterKeyToUse)); } // Invalid request. Like startContainer() with token generated off // old-master-keys. throw new SecretManager.InvalidToken("Given Container " + identifier.GetContainerID ().ToString() + " seems to have an illegally generated token."); } }
public virtual void TestMasterKeyRollOver() { conf.SetLong(YarnConfiguration.RmAmrmTokenMasterKeyRollingIntervalSecs, rolling_interval_sec ); conf.SetLong(YarnConfiguration.RmAmExpiryIntervalMs, am_expire_ms); TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(conf , containerManager); rm.Start(); long startTime = Runtime.CurrentTimeMillis(); Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); ApplicationMasterProtocol rmClient = null; AMRMTokenSecretManager appTokenSecretManager = rm.GetRMContext().GetAMRMTokenSecretManager (); MasterKeyData oldKey = appTokenSecretManager.GetMasterKey(); NUnit.Framework.Assert.IsNotNull(oldKey); try { MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); RMApp app = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < maxWaitAttempts) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); // Create a client to the RM. UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); rmClient.RegisterApplicationMaster(request); // One allocate call. AllocateRequest allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >(); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); // Wait for enough time and make sure the roll_over happens // At mean time, the old AMRMToken should continue to work while (Runtime.CurrentTimeMillis() - startTime < rolling_interval_sec * 1000) { rmClient.Allocate(allocateRequest); Sharpen.Thread.Sleep(500); } MasterKeyData newKey = appTokenSecretManager.GetMasterKey(); NUnit.Framework.Assert.IsNotNull(newKey); NUnit.Framework.Assert.IsFalse("Master key should have changed!", oldKey.Equals(newKey )); // Another allocate call with old AMRMToken. Should continue to work. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); waitCount = 0; while (waitCount++ <= maxWaitAttempts) { if (appTokenSecretManager.GetCurrnetMasterKeyData() != oldKey) { break; } try { rmClient.Allocate(allocateRequest); } catch (Exception) { break; } Sharpen.Thread.Sleep(200); } // active the nextMasterKey, and replace the currentMasterKey NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetCurrnetMasterKeyData().Equals (newKey)); NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetMasterKey().Equals(newKey) ); NUnit.Framework.Assert.IsTrue(appTokenSecretManager.GetNextMasterKeyData() == null ); // Create a new Token Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = appTokenSecretManager .CreateAndGetAMRMToken(applicationAttemptId); SecurityUtil.SetTokenService(newToken, rmBindAddress); currentUser.AddToken(newToken); // Another allocate call. Should continue to work. rpc.StopProxy(rmClient, conf); // To avoid using cached client rmClient = CreateRMClient(rm, conf, rpc, currentUser); allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>( ); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); // Should not work by using the old AMRMToken. rpc.StopProxy(rmClient, conf); // To avoid using cached client try { currentUser.AddToken(amRMToken); rmClient = CreateRMClient(rm, conf, rpc, currentUser); allocateRequest = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>( ); NUnit.Framework.Assert.IsTrue(rmClient.Allocate(allocateRequest).GetAMCommand() == null); NUnit.Framework.Assert.Fail("The old Token should not work"); } catch (Exception) { } } finally { // expect exception rm.Stop(); if (rmClient != null) { rpc.StopProxy(rmClient, conf); } } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> public virtual AllocateResponse Allocate(AllocateRequest request) { AMRMTokenIdentifier amrmTokenIdentifier = AuthorizeRequest(); ApplicationAttemptId appAttemptId = amrmTokenIdentifier.GetApplicationAttemptId(); ApplicationId applicationId = appAttemptId.GetApplicationId(); this.amLivelinessMonitor.ReceivedPing(appAttemptId); /* check if its in cache */ ApplicationMasterService.AllocateResponseLock Lock = responseMap[appAttemptId]; if (Lock == null) { string message = "Application attempt " + appAttemptId + " doesn't exist in ApplicationMasterService cache."; Log.Error(message); throw new ApplicationAttemptNotFoundException(message); } lock (Lock) { AllocateResponse lastResponse = Lock.GetAllocateResponse(); if (!HasApplicationMasterRegistered(appAttemptId)) { string message = "AM is not registered for known application attempt: " + appAttemptId + " or RM had restarted after AM registered . AM should re-register."; Log.Info(message); RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appAttemptId.GetApplicationId ()].GetUser(), RMAuditLogger.AuditConstants.AmAllocate, string.Empty, "ApplicationMasterService" , message, applicationId, appAttemptId); throw new ApplicationMasterNotRegisteredException(message); } if ((request.GetResponseId() + 1) == lastResponse.GetResponseId()) { /* old heartbeat */ return(lastResponse); } else { if (request.GetResponseId() + 1 < lastResponse.GetResponseId()) { string message = "Invalid responseId in AllocateRequest from application attempt: " + appAttemptId + ", expect responseId to be " + (lastResponse.GetResponseId() + 1); throw new InvalidApplicationMasterRequestException(message); } } //filter illegal progress values float filteredProgress = request.GetProgress(); if (float.IsNaN(filteredProgress) || filteredProgress == float.NegativeInfinity || filteredProgress < 0) { request.SetProgress(0); } else { if (filteredProgress > 1 || filteredProgress == float.PositiveInfinity) { request.SetProgress(1); } } // Send the status update to the appAttempt. this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptStatusupdateEvent (appAttemptId, request.GetProgress())); IList <ResourceRequest> ask = request.GetAskList(); IList <ContainerId> release = request.GetReleaseList(); ResourceBlacklistRequest blacklistRequest = request.GetResourceBlacklistRequest(); IList <string> blacklistAdditions = (blacklistRequest != null) ? blacklistRequest. GetBlacklistAdditions() : Sharpen.Collections.EmptyList; IList <string> blacklistRemovals = (blacklistRequest != null) ? blacklistRequest.GetBlacklistRemovals () : Sharpen.Collections.EmptyList; RMApp app = this.rmContext.GetRMApps()[applicationId]; // set label expression for Resource Requests if resourceName=ANY ApplicationSubmissionContext asc = app.GetApplicationSubmissionContext(); foreach (ResourceRequest req in ask) { if (null == req.GetNodeLabelExpression() && ResourceRequest.Any.Equals(req.GetResourceName ())) { req.SetNodeLabelExpression(asc.GetNodeLabelExpression()); } } // sanity check try { RMServerUtils.NormalizeAndValidateRequests(ask, rScheduler.GetMaximumResourceCapability (), app.GetQueue(), rScheduler, rmContext); } catch (InvalidResourceRequestException e) { Log.Warn("Invalid resource ask by application " + appAttemptId, e); throw; } try { RMServerUtils.ValidateBlacklistRequest(blacklistRequest); } catch (InvalidResourceBlacklistRequestException e) { Log.Warn("Invalid blacklist request by application " + appAttemptId, e); throw; } // In the case of work-preserving AM restart, it's possible for the // AM to release containers from the earlier attempt. if (!app.GetApplicationSubmissionContext().GetKeepContainersAcrossApplicationAttempts ()) { try { RMServerUtils.ValidateContainerReleaseRequest(release, appAttemptId); } catch (InvalidContainerReleaseException e) { Log.Warn("Invalid container release by application " + appAttemptId, e); throw; } } // Send new requests to appAttempt. Allocation allocation = this.rScheduler.Allocate(appAttemptId, ask, release, blacklistAdditions , blacklistRemovals); if (!blacklistAdditions.IsEmpty() || !blacklistRemovals.IsEmpty()) { Log.Info("blacklist are updated in Scheduler." + "blacklistAdditions: " + blacklistAdditions + ", " + "blacklistRemovals: " + blacklistRemovals); } RMAppAttempt appAttempt = app.GetRMAppAttempt(appAttemptId); AllocateResponse allocateResponse = recordFactory.NewRecordInstance <AllocateResponse >(); if (!allocation.GetContainers().IsEmpty()) { allocateResponse.SetNMTokens(allocation.GetNMTokens()); } // update the response with the deltas of node status changes IList <RMNode> updatedNodes = new AList <RMNode>(); if (app.PullRMNodeUpdates(updatedNodes) > 0) { IList <NodeReport> updatedNodeReports = new AList <NodeReport>(); foreach (RMNode rmNode in updatedNodes) { SchedulerNodeReport schedulerNodeReport = rScheduler.GetNodeReport(rmNode.GetNodeID ()); Resource used = BuilderUtils.NewResource(0, 0); int numContainers = 0; if (schedulerNodeReport != null) { used = schedulerNodeReport.GetUsedResource(); numContainers = schedulerNodeReport.GetNumContainers(); } NodeId nodeId = rmNode.GetNodeID(); NodeReport report = BuilderUtils.NewNodeReport(nodeId, rmNode.GetState(), rmNode. GetHttpAddress(), rmNode.GetRackName(), used, rmNode.GetTotalCapability(), numContainers , rmNode.GetHealthReport(), rmNode.GetLastHealthReportTime(), rmNode.GetNodeLabels ()); updatedNodeReports.AddItem(report); } allocateResponse.SetUpdatedNodes(updatedNodeReports); } allocateResponse.SetAllocatedContainers(allocation.GetContainers()); allocateResponse.SetCompletedContainersStatuses(appAttempt.PullJustFinishedContainers ()); allocateResponse.SetResponseId(lastResponse.GetResponseId() + 1); allocateResponse.SetAvailableResources(allocation.GetResourceLimit()); allocateResponse.SetNumClusterNodes(this.rScheduler.GetNumClusterNodes()); // add preemption to the allocateResponse message (if any) allocateResponse.SetPreemptionMessage(GeneratePreemptionMessage(allocation)); // update AMRMToken if the token is rolled-up MasterKeyData nextMasterKey = this.rmContext.GetAMRMTokenSecretManager().GetNextMasterKeyData (); if (nextMasterKey != null && nextMasterKey.GetMasterKey().GetKeyId() != amrmTokenIdentifier .GetKeyId()) { RMAppAttemptImpl appAttemptImpl = (RMAppAttemptImpl)appAttempt; Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = appAttempt .GetAMRMToken(); if (nextMasterKey.GetMasterKey().GetKeyId() != appAttemptImpl.GetAMRMTokenKeyId()) { Log.Info("The AMRMToken has been rolled-over. Send new AMRMToken back" + " to application: " + applicationId); amrmToken = rmContext.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId ); appAttemptImpl.SetAMRMToken(amrmToken); } allocateResponse.SetAMRMToken(Org.Apache.Hadoop.Yarn.Api.Records.Token.NewInstance (amrmToken.GetIdentifier(), amrmToken.GetKind().ToString(), amrmToken.GetPassword (), amrmToken.GetService().ToString())); } /* * As we are updating the response inside the lock object so we don't * need to worry about unregister call occurring in between (which * removes the lock object). */ Lock.SetAllocateResponse(allocateResponse); return(allocateResponse); } }
/// <exception cref="System.Exception"/> internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier) { long submitTime = Runtime.CurrentTimeMillis(); long startTime = Runtime.CurrentTimeMillis() + 1234; Configuration conf = new YarnConfiguration(); RMStateStore store = stateStoreHelper.GetRMStateStore(); RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher (); store.SetRMDispatcher(dispatcher); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store); AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager (conf, rmContext)); MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey(); Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData); ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM (); ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001" ); ApplicationId appId1 = attemptId1.GetApplicationId(); StoreApp(store, appId1, submitTime, startTime); verifier.AfterStoreApp(store, appId1); // create application token and client token key for attempt1 Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken (attemptId1, appTokenMgr); SecretKey clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1); ContainerId containerId1 = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001" , appAttemptToken1, clientTokenKey1, dispatcher); string appAttemptIdStr2 = "appattempt_1352994193343_0001_000002"; ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2 ); // create application token and client token key for attempt2 Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken (attemptId2, appTokenMgr); SecretKey clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2); ContainerId containerId2 = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001" , appAttemptToken2, clientTokenKey2, dispatcher); ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001" ); ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId(); StoreApp(store, appIdRemoved, submitTime, startTime); StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null , null, dispatcher); verifier.AfterStoreAppAttempt(store, attemptIdRemoved); RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>(); RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics >(); Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId , RMAppAttempt>(); ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.SetApplicationId(appIdRemoved); Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime); Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn (context); Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts); Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1"); RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>(); Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved ); Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn( mockRmAppAttemptMetrics); Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()). ThenReturn(new AggregateAppResourceUsage(0, 0)); attempts[attemptIdRemoved] = mockRemovedAttempt; store.RemoveApplication(mockRemovedApp); // remove application directory recursively. StoreApp(store, appIdRemoved, submitTime, startTime); StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null , null, dispatcher); store.RemoveApplication(mockRemovedApp); // let things settle down Sharpen.Thread.Sleep(1000); store.Close(); // give tester a chance to modify app state in the store ModifyAppState(); // load state store = stateStoreHelper.GetRMStateStore(); store.SetRMDispatcher(dispatcher); RMStateStore.RMState state = store.LoadState(); IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState (); ApplicationStateData appState = rmAppState[appId1]; // app is loaded NUnit.Framework.Assert.IsNotNull(appState); // app is loaded correctly NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime()); NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime()); // submission context is loaded correctly NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext( ).GetApplicationId()); ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1); // attempt1 is loaded correctly NUnit.Framework.Assert.IsNotNull(attemptState); NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId()); NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus()); // attempt1 container is loaded correctly NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId ()); // attempt1 client token master key is loaded correctly Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName)); attemptState = appState.GetAttempt(attemptId2); // attempt2 is loaded correctly NUnit.Framework.Assert.IsNotNull(attemptState); NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId()); // attempt2 container is loaded correctly NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId ()); // attempt2 client token master key is loaded correctly Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName)); //******* update application/attempt state *******// ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime (), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext (), RMAppState.Finished, "appDiagnostics", 1234); appState2.attempts.PutAll(appState.attempts); store.UpdateApplicationState(appState2); ApplicationAttemptStateData oldAttemptState = attemptState; ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance (oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState .GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished , "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100, oldAttemptState.GetFinishTime(), 0, 0); store.UpdateApplicationAttemptState(newAttemptState); // test updating the state of an app/attempt whose initial state was not // saved. ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10); ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl (); dummyContext.SetApplicationId(dummyAppId); ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime (), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished , "appDiagnostics", 1234); store.UpdateApplicationState(dummyApp); ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId , 6); ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance (dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens (), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime (), 0, 0); store.UpdateApplicationAttemptState(dummyAttempt); // let things settle down Sharpen.Thread.Sleep(1000); store.Close(); // check updated application state. store = stateStoreHelper.GetRMStateStore(); store.SetRMDispatcher(dispatcher); RMStateStore.RMState newRMState = store.LoadState(); IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState (); NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext ().GetApplicationId()]); ApplicationStateData updatedAppState = newRMAppState[appId1]; NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId (), updatedAppState.GetApplicationSubmissionContext().GetApplicationId()); NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime ()); NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime ()); NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser()); // new app state fields NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState()); NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics( )); NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime()); // check updated attempt state NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext ().GetApplicationId()].GetAttempt(dummyAttemptId)); ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState .GetAttemptId()); NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState .GetAttemptId()); NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer ().GetId()); Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName)); // new attempt state fields NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState ()); NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl ()); NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics ()); NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus ()); NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState .GetFinalApplicationStatus()); // assert store is in expected state after everything is cleaned NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid()); store.Close(); }