/// <exception cref="System.IO.IOException"/> private NMTokenIdentifier GetNMTokenId(Org.Apache.Hadoop.Yarn.Api.Records.Token token ) { Org.Apache.Hadoop.Security.Token.Token <NMTokenIdentifier> convertedToken = ConverterUtils .ConvertFromYarn(token, (Text)null); return(convertedToken.DecodeIdentifier()); }
// To avoid using cached client /// <exception cref="System.Exception"/> public virtual void TestAMRMMasterKeysUpdate() { AtomicReference <AMRMTokenSecretManager> spySecretMgrRef = new AtomicReference <AMRMTokenSecretManager >(); MockRM rm = new _MockRM_349(this, spySecretMgrRef, conf); // Skip the login. rm.Start(); MockNM nm = rm.RegisterNode("127.0.0.1:1234", 8000); RMApp app = rm.SubmitApp(200); MockAM am = MockRM.LaunchAndRegisterAM(app, rm, nm); AMRMTokenSecretManager spySecretMgr = spySecretMgrRef.Get(); // Do allocate. Should not update AMRMToken AllocateResponse response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>()); NUnit.Framework.Assert.IsNull(response.GetAMRMToken()); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> oldToken = rm.GetRMContext ().GetRMApps()[app.GetApplicationId()].GetRMAppAttempt(am.GetApplicationAttemptId ()).GetAMRMToken(); // roll over the master key // Do allocate again. the AM should get the latest AMRMToken rm.GetRMContext().GetAMRMTokenSecretManager().RollMasterKey(); response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >()); NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken()); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = ConverterUtils .ConvertFromYarn(response.GetAMRMToken(), new Text(response.GetAMRMToken().GetService ())); NUnit.Framework.Assert.AreEqual(amrmToken.DecodeIdentifier().GetKeyId(), rm.GetRMContext ().GetAMRMTokenSecretManager().GetMasterKey().GetMasterKey().GetKeyId()); // Do allocate again with the same old token and verify the RM sends // back the last generated token instead of generating it again. Org.Mockito.Mockito.Reset(spySecretMgr); UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(am.GetApplicationAttemptId ().ToString(), new string[0]); ugi.AddTokenIdentifier(oldToken.DecodeIdentifier()); response = am.DoAllocateAs(ugi, Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >()); NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken()); Org.Mockito.Mockito.Verify(spySecretMgr, Org.Mockito.Mockito.Never()).CreateAndGetAMRMToken (Matchers.IsA <ApplicationAttemptId>()); // Do allocate again with the updated token and verify we do not // receive a new token to use. response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >()); NUnit.Framework.Assert.IsNull(response.GetAMRMToken()); // Activate the next master key. Since there is new master key generated // in AMRMTokenSecretManager. The AMRMToken will not get updated for AM rm.GetRMContext().GetAMRMTokenSecretManager().ActivateNextMasterKey(); response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >()); NUnit.Framework.Assert.IsNull(response.GetAMRMToken()); rm.Stop(); }
/// <summary>Populate persisted password of AMRMToken back to AMRMTokenSecretManager. /// </summary> /// <exception cref="System.IO.IOException"/> public virtual void AddPersistedPassword(Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier > token) { this.writeLock.Lock(); try { AMRMTokenIdentifier identifier = token.DecodeIdentifier(); Log.Debug("Adding password for " + identifier.GetApplicationAttemptId()); appAttemptSet.AddItem(identifier.GetApplicationAttemptId()); } finally { this.writeLock.Unlock(); } }
/// <summary> /// Test if correct exception (StandbyException or RetriableException) can be /// thrown during the NN failover. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestDelegationTokenDuringNNFailover() { EditLogTailer editLogTailer = nn1.GetNamesystem().GetEditLogTailer(); // stop the editLogTailer of nn1 editLogTailer.Stop(); Configuration conf = (Configuration)Whitebox.GetInternalState(editLogTailer, "conf" ); nn1.GetNamesystem().SetEditLogTailerForTests(new TestDelegationTokensWithHA.EditLogTailerForTest (this, nn1.GetNamesystem(), conf)); // create token Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = GetDelegationToken (fs, "JobTracker"); DelegationTokenIdentifier identifier = new DelegationTokenIdentifier(); byte[] tokenId = token.GetIdentifier(); identifier.ReadFields(new DataInputStream(new ByteArrayInputStream(tokenId))); // Ensure that it's present in the nn0 secret manager and can // be renewed directly from there. Log.Info("A valid token should have non-null password, " + "and should be renewed successfully" ); NUnit.Framework.Assert.IsTrue(null != dtSecretManager.RetrievePassword(identifier )); dtSecretManager.RenewToken(token, "JobTracker"); // transition nn0 to standby cluster.TransitionToStandby(0); try { cluster.GetNameNodeRpc(0).RenewDelegationToken(token); NUnit.Framework.Assert.Fail("StandbyException is expected since nn0 is in standby state" ); } catch (StandbyException e) { GenericTestUtils.AssertExceptionContains(HAServiceProtocol.HAServiceState.Standby .ToString(), e); } new _Thread_220().Start(); Sharpen.Thread.Sleep(1000); try { nn1.GetNamesystem().VerifyToken(token.DecodeIdentifier(), token.GetPassword()); NUnit.Framework.Assert.Fail("RetriableException/StandbyException is expected since nn1 is in transition" ); } catch (IOException e) { NUnit.Framework.Assert.IsTrue(e is StandbyException || e is RetriableException); Log.Info("Got expected exception", e); } catchup = true; lock (this) { Sharpen.Runtime.NotifyAll(this); } Configuration clientConf = dfs.GetConf(); DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Renew ); DoRenewOrCancel(token, clientConf, TestDelegationTokensWithHA.TokenTestAction.Cancel ); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private void VerifyNewVersionToken(Configuration conf, TestClientToAMTokens.CustomAM am, Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token, MockRM rm) { UserGroupInformation ugi; ugi = UserGroupInformation.CreateRemoteUser("me"); Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> newToken = new Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier>(new ClientToAMTokenIdentifierForTest (token.DecodeIdentifier(), "message"), am.GetClientToAMTokenSecretManager()); newToken.SetService(token.GetService()); ugi.AddToken(newToken); ugi.DoAs(new _PrivilegedExceptionAction_386(am, conf)); }
/// <summary>Decode the token identifier.</summary> /// <remarks> /// Decode the token identifier. The subclass can customize the way to decode /// the token identifier. /// </remarks> /// <param name="token">the token where to extract the identifier</param> /// <returns>the delegation token identifier</returns> /// <exception cref="System.IO.IOException"/> public virtual TokenIdent DecodeTokenIdentifier(Org.Apache.Hadoop.Security.Token.Token <TokenIdent> token) { return(token.DecodeIdentifier()); }
// Test does major 6 steps verification. // Step-1 : AMRMClient send allocate request for 2 container requests // Step-2 : 2 containers are allocated by RM. // Step-3 : AM Send 1 containerRequest(cRequest3) and 1 releaseRequests to // RM // Step-4 : On RM restart, AM(does not know RM is restarted) sends additional // containerRequest(cRequest4) and blacklisted nodes. // Intern RM send resync command // Step-5 : Allocater after resync command & new containerRequest(cRequest5) // Step-6 : RM allocates containers i.e cRequest3,cRequest4 and cRequest5 /// <exception cref="System.Exception"/> public virtual void TestAMRMClientResendsRequestsOnRMRestart() { UserGroupInformation.SetLoginUser(null); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); // Phase-1 Start 1st RM TestAMRMClientOnRMRestart.MyResourceManager rm1 = new TestAMRMClientOnRMRestart.MyResourceManager (conf, memStore); rm1.Start(); DrainDispatcher dispatcher = (DrainDispatcher)rm1.GetRMContext().GetDispatcher(); // Submit the application RMApp app = rm1.SubmitApp(1024); dispatcher.Await(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.GetResourceTrackerService()); nm1.RegisterNode(); nm1.NodeHeartbeat(true); // Node heartbeat dispatcher.Await(); ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId(); rm1.SendAMLaunched(appAttemptId); dispatcher.Await(); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm1.GetRMContext ().GetRMApps()[appAttemptId.GetApplicationId()].GetRMAppAttempt(appAttemptId).GetAMRMToken (); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); ugi.AddTokenIdentifier(token.DecodeIdentifier()); // Step-1 : AMRMClient send allocate request for 2 ContainerRequest // cRequest1 = h1 and cRequest2 = h1,h2 // blacklisted nodes = h2 AMRMClient <AMRMClient.ContainerRequest> amClient = new TestAMRMClientOnRMRestart.MyAMRMClientImpl (rm1); amClient.Init(conf); amClient.Start(); amClient.RegisterApplicationMaster("Host", 10000, string.Empty); AMRMClient.ContainerRequest cRequest1 = CreateReq(1, 1024, new string[] { "h1" }); amClient.AddContainerRequest(cRequest1); AMRMClient.ContainerRequest cRequest2 = CreateReq(1, 1024, new string[] { "h1", "h2" }); amClient.AddContainerRequest(cRequest2); IList <string> blacklistAdditions = new AList <string>(); IList <string> blacklistRemoval = new AList <string>(); blacklistAdditions.AddItem("h2"); blacklistRemoval.AddItem("h10"); amClient.UpdateBlacklist(blacklistAdditions, blacklistRemoval); blacklistAdditions.Remove("h2"); // remove from local list AllocateResponse allocateResponse = amClient.Allocate(0.1f); dispatcher.Await(); NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse .GetAllocatedContainers().Count); // Why 4 ask, why not 3 ask even h2 is blacklisted? // On blacklisting host,applicationmaster has to remove ask request from // remoterequest table.Here,test does not remove explicitely AssertAsksAndReleases(4, 0, rm1); AssertBlacklistAdditionsAndRemovals(1, 1, rm1); // Step-2 : NM heart beat is sent. // On 2nd AM allocate request, RM allocates 2 containers to AM nm1.NodeHeartbeat(true); // Node heartbeat dispatcher.Await(); allocateResponse = amClient.Allocate(0.2f); dispatcher.Await(); // 2 containers are allocated i.e for cRequest1 and cRequest2. NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 2, allocateResponse .GetAllocatedContainers().Count); AssertAsksAndReleases(0, 0, rm1); AssertBlacklistAdditionsAndRemovals(0, 0, rm1); IList <Container> allocatedContainers = allocateResponse.GetAllocatedContainers(); // removed allocated container requests amClient.RemoveContainerRequest(cRequest1); amClient.RemoveContainerRequest(cRequest2); allocateResponse = amClient.Allocate(0.2f); dispatcher.Await(); NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse .GetAllocatedContainers().Count); AssertAsksAndReleases(4, 0, rm1); AssertBlacklistAdditionsAndRemovals(0, 0, rm1); // Step-3 : Send 1 containerRequest and 1 releaseRequests to RM AMRMClient.ContainerRequest cRequest3 = CreateReq(1, 1024, new string[] { "h1" }); amClient.AddContainerRequest(cRequest3); int pendingRelease = 0; IEnumerator <Container> it = allocatedContainers.GetEnumerator(); while (it.HasNext()) { amClient.ReleaseAssignedContainer(it.Next().GetId()); pendingRelease++; it.Remove(); break; } // remove one container allocateResponse = amClient.Allocate(0.3f); dispatcher.Await(); NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse .GetAllocatedContainers().Count); AssertAsksAndReleases(3, pendingRelease, rm1); AssertBlacklistAdditionsAndRemovals(0, 0, rm1); int completedContainer = allocateResponse.GetCompletedContainersStatuses().Count; pendingRelease -= completedContainer; // Phase-2 start 2nd RM is up TestAMRMClientOnRMRestart.MyResourceManager rm2 = new TestAMRMClientOnRMRestart.MyResourceManager (conf, memStore); rm2.Start(); nm1.SetResourceTrackerService(rm2.GetResourceTrackerService()); ((TestAMRMClientOnRMRestart.MyAMRMClientImpl)amClient).UpdateRMProxy(rm2); dispatcher = (DrainDispatcher)rm2.GetRMContext().GetDispatcher(); // NM should be rebooted on heartbeat, even first heartbeat for nm2 NodeHeartbeatResponse hbResponse = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual(NodeAction.Resync, hbResponse.GetNodeAction()); // new NM to represent NM re-register nm1 = new MockNM("h1:1234", 10240, rm2.GetResourceTrackerService()); nm1.RegisterNode(); nm1.NodeHeartbeat(true); dispatcher.Await(); blacklistAdditions.AddItem("h3"); amClient.UpdateBlacklist(blacklistAdditions, null); blacklistAdditions.Remove("h3"); it = allocatedContainers.GetEnumerator(); while (it.HasNext()) { amClient.ReleaseAssignedContainer(it.Next().GetId()); pendingRelease++; it.Remove(); } AMRMClient.ContainerRequest cRequest4 = CreateReq(1, 1024, new string[] { "h1", "h2" }); amClient.AddContainerRequest(cRequest4); // Step-4 : On RM restart, AM(does not know RM is restarted) sends // additional // containerRequest and blacklisted nodes. // Intern RM send resync command,AMRMClient resend allocate request allocateResponse = amClient.Allocate(0.3f); dispatcher.Await(); completedContainer = allocateResponse.GetCompletedContainersStatuses().Count; pendingRelease -= completedContainer; AssertAsksAndReleases(4, pendingRelease, rm2); AssertBlacklistAdditionsAndRemovals(2, 0, rm2); AMRMClient.ContainerRequest cRequest5 = CreateReq(1, 1024, new string[] { "h1", "h2" , "h3" }); amClient.AddContainerRequest(cRequest5); // Step-5 : Allocater after resync command allocateResponse = amClient.Allocate(0.5f); dispatcher.Await(); NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse .GetAllocatedContainers().Count); AssertAsksAndReleases(5, 0, rm2); AssertBlacklistAdditionsAndRemovals(0, 0, rm2); int noAssignedContainer = 0; int count = 5; while (count-- > 0) { nm1.NodeHeartbeat(true); dispatcher.Await(); allocateResponse = amClient.Allocate(0.5f); dispatcher.Await(); noAssignedContainer += allocateResponse.GetAllocatedContainers().Count; if (noAssignedContainer == 3) { break; } Sharpen.Thread.Sleep(1000); } // Step-6 : RM allocates containers i.e cRequest3,cRequest4 and cRequest5 NUnit.Framework.Assert.AreEqual("Number of container should be 3", 3, noAssignedContainer ); amClient.Stop(); rm1.Stop(); rm2.Stop(); }
// Test verify for AM issued with rolled-over AMRMToken // is still able to communicate with restarted RM. /// <exception cref="System.Exception"/> public virtual void TestAMRMClientOnAMRMTokenRollOverOnRMRestart() { conf.SetLong(YarnConfiguration.RmAmrmTokenMasterKeyRollingIntervalSecs, rolling_interval_sec ); conf.SetLong(YarnConfiguration.RmAmExpiryIntervalMs, am_expire_ms); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); // start first RM TestAMRMClientOnRMRestart.MyResourceManager2 rm1 = new TestAMRMClientOnRMRestart.MyResourceManager2 (conf, memStore); rm1.Start(); DrainDispatcher dispatcher = (DrainDispatcher)rm1.GetRMContext().GetDispatcher(); long startTime = Runtime.CurrentTimeMillis(); // Submit the application RMApp app = rm1.SubmitApp(1024); dispatcher.Await(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.GetResourceTrackerService()); nm1.RegisterNode(); nm1.NodeHeartbeat(true); // Node heartbeat dispatcher.Await(); ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId(); rm1.SendAMLaunched(appAttemptId); dispatcher.Await(); AMRMTokenSecretManager amrmTokenSecretManagerForRM1 = rm1.GetRMContext().GetAMRMTokenSecretManager (); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = amrmTokenSecretManagerForRM1 .CreateAndGetAMRMToken(appAttemptId); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); ugi.AddTokenIdentifier(token.DecodeIdentifier()); AMRMClient <AMRMClient.ContainerRequest> amClient = new TestAMRMClientOnRMRestart.MyAMRMClientImpl (rm1); amClient.Init(conf); amClient.Start(); amClient.RegisterApplicationMaster("h1", 10000, string.Empty); amClient.Allocate(0.1f); // Wait for enough time and make sure the roll_over happens // At mean time, the old AMRMToken should continue to work while (Runtime.CurrentTimeMillis() - startTime < rolling_interval_sec * 1000) { amClient.Allocate(0.1f); try { Sharpen.Thread.Sleep(1000); } catch (Exception) { } } // DO NOTHING NUnit.Framework.Assert.IsTrue(amrmTokenSecretManagerForRM1.GetMasterKey().GetMasterKey ().GetKeyId() != token.DecodeIdentifier().GetKeyId()); amClient.Allocate(0.1f); // active the nextMasterKey, and replace the currentMasterKey Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> newToken = amrmTokenSecretManagerForRM1 .CreateAndGetAMRMToken(appAttemptId); int waitCount = 0; while (waitCount++ <= 50) { if (amrmTokenSecretManagerForRM1.GetCurrnetMasterKeyData().GetMasterKey().GetKeyId () != token.DecodeIdentifier().GetKeyId()) { break; } try { amClient.Allocate(0.1f); } catch (Exception) { break; } Sharpen.Thread.Sleep(500); } NUnit.Framework.Assert.IsTrue(amrmTokenSecretManagerForRM1.GetNextMasterKeyData() == null); NUnit.Framework.Assert.IsTrue(amrmTokenSecretManagerForRM1.GetCurrnetMasterKeyData ().GetMasterKey().GetKeyId() == newToken.DecodeIdentifier().GetKeyId()); // start 2nd RM conf.Set(YarnConfiguration.RmSchedulerAddress, "0.0.0.0:9030"); TestAMRMClientOnRMRestart.MyResourceManager2 rm2 = new TestAMRMClientOnRMRestart.MyResourceManager2 (conf, memStore); rm2.Start(); nm1.SetResourceTrackerService(rm2.GetResourceTrackerService()); ((TestAMRMClientOnRMRestart.MyAMRMClientImpl)amClient).UpdateRMProxy(rm2); dispatcher = (DrainDispatcher)rm2.GetRMContext().GetDispatcher(); AMRMTokenSecretManager amrmTokenSecretManagerForRM2 = rm2.GetRMContext().GetAMRMTokenSecretManager (); NUnit.Framework.Assert.IsTrue(amrmTokenSecretManagerForRM2.GetCurrnetMasterKeyData ().GetMasterKey().GetKeyId() == newToken.DecodeIdentifier().GetKeyId()); NUnit.Framework.Assert.IsTrue(amrmTokenSecretManagerForRM2.GetNextMasterKeyData() == null); try { UserGroupInformation testUser = UserGroupInformation.CreateRemoteUser("testUser"); SecurityUtil.SetTokenService(token, rm2.GetApplicationMasterService().GetBindAddress ()); testUser.AddToken(token); testUser.DoAs(new _PrivilegedAction_480(rm2)).Allocate(Org.Apache.Hadoop.Yarn.Util.Records .NewRecord <AllocateRequest>()); NUnit.Framework.Assert.Fail("The old Token should not work"); } catch (Exception ex) { NUnit.Framework.Assert.IsTrue(ex is SecretManager.InvalidToken); NUnit.Framework.Assert.IsTrue(ex.Message.Contains("Invalid AMRMToken from " + token .DecodeIdentifier().GetApplicationAttemptId())); } // make sure the recovered AMRMToken works for new RM amClient.Allocate(0.1f); amClient.UnregisterApplicationMaster(FinalApplicationStatus.Succeeded, null, null ); amClient.Stop(); rm1.Stop(); rm2.Stop(); }
// Test verify for // 1. AM try to unregister without registering // 2. AM register to RM, and try to unregister immediately after RM restart /// <exception cref="System.Exception"/> public virtual void TestAMRMClientForUnregisterAMOnRMRestart() { MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); // Phase-1 Start 1st RM TestAMRMClientOnRMRestart.MyResourceManager rm1 = new TestAMRMClientOnRMRestart.MyResourceManager (conf, memStore); rm1.Start(); DrainDispatcher dispatcher = (DrainDispatcher)rm1.GetRMContext().GetDispatcher(); // Submit the application RMApp app = rm1.SubmitApp(1024); dispatcher.Await(); MockNM nm1 = new MockNM("h1:1234", 15120, rm1.GetResourceTrackerService()); nm1.RegisterNode(); nm1.NodeHeartbeat(true); // Node heartbeat dispatcher.Await(); ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId(); rm1.SendAMLaunched(appAttemptId); dispatcher.Await(); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm1.GetRMContext ().GetRMApps()[appAttemptId.GetApplicationId()].GetRMAppAttempt(appAttemptId).GetAMRMToken (); UserGroupInformation ugi = UserGroupInformation.GetCurrentUser(); ugi.AddTokenIdentifier(token.DecodeIdentifier()); AMRMClient <AMRMClient.ContainerRequest> amClient = new TestAMRMClientOnRMRestart.MyAMRMClientImpl (rm1); amClient.Init(conf); amClient.Start(); amClient.RegisterApplicationMaster("h1", 10000, string.Empty); amClient.Allocate(0.1f); // Phase-2 start 2nd RM is up TestAMRMClientOnRMRestart.MyResourceManager rm2 = new TestAMRMClientOnRMRestart.MyResourceManager (conf, memStore); rm2.Start(); nm1.SetResourceTrackerService(rm2.GetResourceTrackerService()); ((TestAMRMClientOnRMRestart.MyAMRMClientImpl)amClient).UpdateRMProxy(rm2); dispatcher = (DrainDispatcher)rm2.GetRMContext().GetDispatcher(); // NM should be rebooted on heartbeat, even first heartbeat for nm2 NodeHeartbeatResponse hbResponse = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual(NodeAction.Resync, hbResponse.GetNodeAction()); // new NM to represent NM re-register nm1 = new MockNM("h1:1234", 10240, rm2.GetResourceTrackerService()); ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 1); NMContainerStatus containerReport = NMContainerStatus.NewInstance(containerId, ContainerState .Running, Resource.NewInstance(1024, 1), "recover container", 0, Priority.NewInstance (0), 0); nm1.RegisterNode(Arrays.AsList(containerReport), null); nm1.NodeHeartbeat(true); dispatcher.Await(); amClient.UnregisterApplicationMaster(FinalApplicationStatus.Succeeded, null, null ); rm2.WaitForState(appAttemptId, RMAppAttemptState.Finishing); nm1.NodeHeartbeat(appAttemptId, 1, ContainerState.Complete); rm2.WaitForState(appAttemptId, RMAppAttemptState.Finished); rm2.WaitForState(app.GetApplicationId(), RMAppState.Finished); amClient.Stop(); rm1.Stop(); rm2.Stop(); }