/// <exception cref="System.Exception"/> public virtual void TestAMRMTokenSecretManagerStateStore(RMStateStoreTestBase.RMStateStoreHelper stateStoreHelper) { System.Console.Out.WriteLine("Start testing"); RMStateStore store = stateStoreHelper.GetRMStateStore(); RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher (); store.SetRMDispatcher(dispatcher); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store); Configuration conf = new YarnConfiguration(); AMRMTokenSecretManager appTokenMgr = new AMRMTokenSecretManager(conf, rmContext); //create and save the first masterkey MasterKeyData firstMasterKeyData = appTokenMgr.CreateNewMasterKey(); AMRMTokenSecretManagerState state1 = AMRMTokenSecretManagerState.NewInstance(firstMasterKeyData .GetMasterKey(), null); rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state1, false); // load state store = stateStoreHelper.GetRMStateStore(); Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store); store.SetRMDispatcher(dispatcher); RMStateStore.RMState state = store.LoadState(); NUnit.Framework.Assert.IsNotNull(state.GetAMRMTokenSecretManagerState()); NUnit.Framework.Assert.AreEqual(firstMasterKeyData.GetMasterKey(), state.GetAMRMTokenSecretManagerState ().GetCurrentMasterKey()); NUnit.Framework.Assert.IsNull(state.GetAMRMTokenSecretManagerState().GetNextMasterKey ()); //create and save the second masterkey MasterKeyData secondMasterKeyData = appTokenMgr.CreateNewMasterKey(); AMRMTokenSecretManagerState state2 = AMRMTokenSecretManagerState.NewInstance(firstMasterKeyData .GetMasterKey(), secondMasterKeyData.GetMasterKey()); rmContext.GetStateStore().StoreOrUpdateAMRMTokenSecretManager(state2, true); // load state store = stateStoreHelper.GetRMStateStore(); Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store); store.SetRMDispatcher(dispatcher); RMStateStore.RMState state_2 = store.LoadState(); NUnit.Framework.Assert.IsNotNull(state_2.GetAMRMTokenSecretManagerState()); NUnit.Framework.Assert.AreEqual(firstMasterKeyData.GetMasterKey(), state_2.GetAMRMTokenSecretManagerState ().GetCurrentMasterKey()); NUnit.Framework.Assert.AreEqual(secondMasterKeyData.GetMasterKey(), state_2.GetAMRMTokenSecretManagerState ().GetNextMasterKey()); // re-create the masterKeyData based on the recovered masterkey // should have the same secretKey appTokenMgr.Recover(state_2); NUnit.Framework.Assert.AreEqual(appTokenMgr.GetCurrnetMasterKeyData().GetSecretKey (), firstMasterKeyData.GetSecretKey()); NUnit.Framework.Assert.AreEqual(appTokenMgr.GetNextMasterKeyData().GetSecretKey() , secondMasterKeyData.GetSecretKey()); store.Close(); }
/// <exception cref="System.Exception"/> public virtual void TestZKSessionTimeout() { TestZKRMStateStoreZKClientConnections.TestZKClient zkClientTester = new TestZKRMStateStoreZKClientConnections.TestZKClient (this); string path = "/test"; YarnConfiguration conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmZkTimeoutMs, ZkTimeoutMs); ZKRMStateStore store = (ZKRMStateStore)zkClientTester.GetRMStateStore(conf); RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher (); store.SetRMDispatcher(dispatcher); // a hack to trigger expired event zkClientTester.forExpire = true; // trigger watch store.CreateWithRetries(path, null, ZooDefs.Ids.OpenAclUnsafe, CreateMode.Persistent ); store.GetDataWithRetries(path, true); store.SetDataWithRetries(path, Sharpen.Runtime.GetBytesForString("bytes"), 0); zkClientTester.syncBarrier.Await(); // after this point, expired event has already been processed. try { byte[] ret = store.GetDataWithRetries(path, false); NUnit.Framework.Assert.AreEqual("bytes", Sharpen.Runtime.GetStringForBytes(ret)); } catch (Exception e) { string error = "New session creation failed"; Log.Error(error, e); NUnit.Framework.Assert.Fail(error); } // send Disconnected event from old client session to ZKRMStateStore // check the current client session is not affected. NUnit.Framework.Assert.IsTrue(zkClientTester.oldWatcher != null); WatchedEvent disconnectedEvent = new WatchedEvent(Watcher.Event.EventType.None, Watcher.Event.KeeperState .Disconnected, null); zkClientTester.oldWatcher.Process(disconnectedEvent); NUnit.Framework.Assert.IsTrue(store.zkClient != null); zkClientTester.watcher.Process(disconnectedEvent); NUnit.Framework.Assert.IsTrue(store.zkClient == null); WatchedEvent connectedEvent = new WatchedEvent(Watcher.Event.EventType.None, Watcher.Event.KeeperState .SyncConnected, null); zkClientTester.watcher.Process(connectedEvent); NUnit.Framework.Assert.IsTrue(store.zkClient != null); NUnit.Framework.Assert.IsTrue(store.zkClient == store.activeZkClient); }
/// <exception cref="System.Exception"/> public virtual void TestZKClientDisconnectAndReconnect() { TestZKRMStateStoreZKClientConnections.TestZKClient zkClientTester = new TestZKRMStateStoreZKClientConnections.TestZKClient (this); string path = "/test"; YarnConfiguration conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmZkTimeoutMs, ZkTimeoutMs); ZKRMStateStore store = (ZKRMStateStore)zkClientTester.GetRMStateStore(conf); RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher (); store.SetRMDispatcher(dispatcher); // trigger watch store.CreateWithRetries(path, null, ZooDefs.Ids.OpenAclUnsafe, CreateMode.Persistent ); store.GetDataWithRetries(path, true); store.SetDataWithRetries(path, Sharpen.Runtime.GetBytesForString("newBytes"), 0); StopServer(); AtomicBoolean isSucceeded = new AtomicBoolean(false); zkClientTester.watcher.WaitForDisconnected(ZkOpWaitTime); Sharpen.Thread thread = new _Thread_190(store, path, isSucceeded); thread.Start(); // ZKRMStateStore Session restored StartServer(); zkClientTester.watcher.WaitForConnected(ZkOpWaitTime); byte[] ret = null; try { ret = store.GetDataWithRetries(path, true); } catch (Exception e) { string error = "ZKRMStateStore Session restore failed"; Log.Error(error, e); NUnit.Framework.Assert.Fail(error); } NUnit.Framework.Assert.AreEqual("newBytes", Sharpen.Runtime.GetStringForBytes(ret )); thread.Join(); NUnit.Framework.Assert.IsTrue(isSucceeded.Get()); }
/// <exception cref="System.Exception"/> public virtual void TestZKClientRetry() { TestZKRMStateStoreZKClientConnections.TestZKClient zkClientTester = new TestZKRMStateStoreZKClientConnections.TestZKClient (this); string path = "/test"; YarnConfiguration conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmZkTimeoutMs, ZkTimeoutMs); conf.SetLong(YarnConfiguration.RmZkRetryIntervalMs, 100); ZKRMStateStore store = (ZKRMStateStore)zkClientTester.GetRMStateStore(conf); RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher (); store.SetRMDispatcher(dispatcher); AtomicBoolean assertionFailedInThread = new AtomicBoolean(false); StopServer(); Sharpen.Thread clientThread = new _Thread_151(store, path, assertionFailedInThread ); Sharpen.Thread.Sleep(2000); StartServer(); clientThread.Join(); NUnit.Framework.Assert.IsFalse(assertionFailedInThread.Get()); }
public virtual void TestDuplicateRMAppDeletion() { TestZKRMStateStore.TestZKRMStateStoreTester zkTester = new TestZKRMStateStore.TestZKRMStateStoreTester (this); long submitTime = Runtime.CurrentTimeMillis(); long startTime = Runtime.CurrentTimeMillis() + 1234; RMStateStore store = zkTester.GetRMStateStore(); RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher (); store.SetRMDispatcher(dispatcher); ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001" ); ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId(); StoreApp(store, appIdRemoved, submitTime, startTime); StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null , null, dispatcher); ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.SetApplicationId(appIdRemoved); ApplicationStateData appStateRemoved = ApplicationStateData.NewInstance(submitTime , startTime, context, "user1"); appStateRemoved.attempts[attemptIdRemoved] = null; store.RemoveApplicationStateInternal(appStateRemoved); try { store.RemoveApplicationStateInternal(appStateRemoved); } catch (KeeperException.NoNodeException) { NUnit.Framework.Assert.Fail("NoNodeException should not happen."); } store.Close(); }
internal virtual void WaitNotify(RMStateStoreTestBase.TestDispatcher dispatcher) { long startTime = Runtime.CurrentTimeMillis(); while (!dispatcher.notified) { lock (dispatcher) { try { Sharpen.Runtime.Wait(dispatcher, 1000); } catch (Exception e) { Sharpen.Runtime.PrintStackTrace(e); } } if (Runtime.CurrentTimeMillis() - startTime > 1000 * 60) { NUnit.Framework.Assert.Fail("Timed out attempt store notification"); } } dispatcher.notified = false; }
public virtual int Run(string[] args) { Log.Info("Starting ZKRMStateStorePerf ver." + version); int numApp = ZkPerfNumAppDefault; int numAppAttemptPerApp = ZkPerfNumAppattemptPerApp; string hostPort = null; bool launchLocalZK = true; if (args.Length == 0) { System.Console.Error.WriteLine("Missing arguments."); return(-1); } for (int i = 0; i < args.Length; i++) { // parse command line if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-appsize")) { numApp = System.Convert.ToInt32(args[++i]); } else { if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-appattemptsize")) { numAppAttemptPerApp = System.Convert.ToInt32(args[++i]); } else { if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-hostPort")) { hostPort = args[++i]; launchLocalZK = false; } else { if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-workingZnode")) { workingZnode = args[++i]; } else { System.Console.Error.WriteLine("Illegal argument: " + args[i]); return(-1); } } } } } if (launchLocalZK) { try { SetUp(); } catch (Exception e) { System.Console.Error.WriteLine("failed to setup. : " + e.Message); return(-1); } } InitStore(hostPort); long submitTime = Runtime.CurrentTimeMillis(); long startTime = Runtime.CurrentTimeMillis() + 1234; AList <ApplicationId> applicationIds = new AList <ApplicationId>(); AList <RMApp> rmApps = new AList <RMApp>(); AList <ApplicationAttemptId> attemptIds = new AList <ApplicationAttemptId>(); Dictionary <ApplicationId, ICollection <ApplicationAttemptId> > appIdsToAttemptId = new Dictionary <ApplicationId, ICollection <ApplicationAttemptId> >(); RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher (); store.SetRMDispatcher(dispatcher); for (int i_1 = 0; i_1 < numApp; i_1++) { ApplicationId appId = ApplicationId.NewInstance(clusterTimeStamp, i_1); applicationIds.AddItem(appId); AList <ApplicationAttemptId> attemptIdsForThisApp = new AList <ApplicationAttemptId >(); for (int j = 0; j < numAppAttemptPerApp; j++) { ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, j); attemptIdsForThisApp.AddItem(attemptId); } appIdsToAttemptId[appId] = new LinkedHashSet(attemptIdsForThisApp); Sharpen.Collections.AddAll(attemptIds, attemptIdsForThisApp); } foreach (ApplicationId appId_1 in applicationIds) { RMApp app = null; try { app = StoreApp(store, appId_1, submitTime, startTime); } catch (Exception e) { System.Console.Error.WriteLine("failed to create Application Znode. : " + e.Message ); return(-1); } WaitNotify(dispatcher); rmApps.AddItem(app); } foreach (ApplicationAttemptId attemptId_1 in attemptIds) { Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> tokenId = GenerateAMRMToken (attemptId_1, appTokenMgr); SecretKey clientTokenKey = clientToAMTokenMgr.CreateMasterKey(attemptId_1); try { StoreAttempt(store, attemptId_1, ContainerId.NewContainerId(attemptId_1, 0L).ToString (), tokenId, clientTokenKey, dispatcher); } catch (Exception e) { System.Console.Error.WriteLine("failed to create AppAttempt Znode. : " + e.Message ); return(-1); } } long storeStart = Runtime.CurrentTimeMillis(); try { store.LoadState(); } catch (Exception e) { System.Console.Error.WriteLine("failed to locaState from ZKRMStateStore. : " + e. Message); return(-1); } long storeEnd = Runtime.CurrentTimeMillis(); long loadTime = storeEnd - storeStart; string resultMsg = "ZKRMStateStore takes " + loadTime + " msec to loadState."; Log.Info(resultMsg); System.Console.Out.WriteLine(resultMsg); // cleanup try { foreach (RMApp app in rmApps) { ApplicationStateData appState = ApplicationStateData.NewInstance(app.GetSubmitTime (), app.GetStartTime(), app.GetApplicationSubmissionContext(), app.GetUser()); ApplicationId appId = app.GetApplicationId(); IDictionary m = Org.Mockito.Mockito.Mock <IDictionary>(); Org.Mockito.Mockito.When(m.Keys).ThenReturn(appIdsToAttemptId[appId_1]); appState.attempts = m; store.RemoveApplicationStateInternal(appState); } } catch (Exception e) { System.Console.Error.WriteLine("failed to cleanup. : " + e.Message); return(-1); } return(0); }
/// <exception cref="System.Exception"/> public virtual void TestRMDTSecretManagerStateStore(RMStateStoreTestBase.RMStateStoreHelper stateStoreHelper) { RMStateStore store = stateStoreHelper.GetRMStateStore(); RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher (); store.SetRMDispatcher(dispatcher); // store RM delegation token; RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(new Text("owner1" ), new Text("renewer1"), new Text("realuser1")); int sequenceNumber = 1111; dtId1.SetSequenceNumber(sequenceNumber); byte[] tokenBeforeStore = dtId1.GetBytes(); long renewDate1 = Runtime.CurrentTimeMillis(); store.StoreRMDelegationToken(dtId1, renewDate1); ModifyRMDelegationTokenState(); IDictionary <RMDelegationTokenIdentifier, long> token1 = new Dictionary <RMDelegationTokenIdentifier , long>(); token1[dtId1] = renewDate1; // store delegation key; DelegationKey key = new DelegationKey(1234, 4321, Sharpen.Runtime.GetBytesForString ("keyBytes")); HashSet <DelegationKey> keySet = new HashSet <DelegationKey>(); keySet.AddItem(key); store.StoreRMDTMasterKey(key); RMStateStore.RMDTSecretManagerState secretManagerState = store.LoadState().GetRMDTSecretManagerState (); NUnit.Framework.Assert.AreEqual(token1, secretManagerState.GetTokenState()); NUnit.Framework.Assert.AreEqual(keySet, secretManagerState.GetMasterKeyState()); NUnit.Framework.Assert.AreEqual(sequenceNumber, secretManagerState.GetDTSequenceNumber ()); RMDelegationTokenIdentifier tokenAfterStore = secretManagerState.GetTokenState(). Keys.GetEnumerator().Next(); NUnit.Framework.Assert.IsTrue(Arrays.Equals(tokenBeforeStore, tokenAfterStore.GetBytes ())); // update RM delegation token; renewDate1 = Runtime.CurrentTimeMillis(); store.UpdateRMDelegationToken(dtId1, renewDate1); token1[dtId1] = renewDate1; RMStateStore.RMDTSecretManagerState updateSecretManagerState = store.LoadState(). GetRMDTSecretManagerState(); NUnit.Framework.Assert.AreEqual(token1, updateSecretManagerState.GetTokenState()); NUnit.Framework.Assert.AreEqual(keySet, updateSecretManagerState.GetMasterKeyState ()); NUnit.Framework.Assert.AreEqual(sequenceNumber, updateSecretManagerState.GetDTSequenceNumber ()); // check to delete delegationKey store.RemoveRMDTMasterKey(key); keySet.Clear(); RMStateStore.RMDTSecretManagerState noKeySecretManagerState = store.LoadState().GetRMDTSecretManagerState (); NUnit.Framework.Assert.AreEqual(token1, noKeySecretManagerState.GetTokenState()); NUnit.Framework.Assert.AreEqual(keySet, noKeySecretManagerState.GetMasterKeyState ()); NUnit.Framework.Assert.AreEqual(sequenceNumber, noKeySecretManagerState.GetDTSequenceNumber ()); // check to delete delegationToken store.RemoveRMDelegationToken(dtId1); RMStateStore.RMDTSecretManagerState noKeyAndTokenSecretManagerState = store.LoadState ().GetRMDTSecretManagerState(); token1.Clear(); NUnit.Framework.Assert.AreEqual(token1, noKeyAndTokenSecretManagerState.GetTokenState ()); NUnit.Framework.Assert.AreEqual(keySet, noKeyAndTokenSecretManagerState.GetMasterKeyState ()); NUnit.Framework.Assert.AreEqual(sequenceNumber, noKeySecretManagerState.GetDTSequenceNumber ()); store.Close(); }
/// <exception cref="System.Exception"/> internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier) { long submitTime = Runtime.CurrentTimeMillis(); long startTime = Runtime.CurrentTimeMillis() + 1234; Configuration conf = new YarnConfiguration(); RMStateStore store = stateStoreHelper.GetRMStateStore(); RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher (); store.SetRMDispatcher(dispatcher); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store); AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager (conf, rmContext)); MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey(); Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData); ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM (); ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001" ); ApplicationId appId1 = attemptId1.GetApplicationId(); StoreApp(store, appId1, submitTime, startTime); verifier.AfterStoreApp(store, appId1); // create application token and client token key for attempt1 Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken (attemptId1, appTokenMgr); SecretKey clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1); ContainerId containerId1 = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001" , appAttemptToken1, clientTokenKey1, dispatcher); string appAttemptIdStr2 = "appattempt_1352994193343_0001_000002"; ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2 ); // create application token and client token key for attempt2 Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken (attemptId2, appTokenMgr); SecretKey clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2); ContainerId containerId2 = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001" , appAttemptToken2, clientTokenKey2, dispatcher); ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001" ); ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId(); StoreApp(store, appIdRemoved, submitTime, startTime); StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null , null, dispatcher); verifier.AfterStoreAppAttempt(store, attemptIdRemoved); RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>(); RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics >(); Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId , RMAppAttempt>(); ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl(); context.SetApplicationId(appIdRemoved); Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime); Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn (context); Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts); Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1"); RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>(); Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved ); Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn( mockRmAppAttemptMetrics); Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()). ThenReturn(new AggregateAppResourceUsage(0, 0)); attempts[attemptIdRemoved] = mockRemovedAttempt; store.RemoveApplication(mockRemovedApp); // remove application directory recursively. StoreApp(store, appIdRemoved, submitTime, startTime); StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null , null, dispatcher); store.RemoveApplication(mockRemovedApp); // let things settle down Sharpen.Thread.Sleep(1000); store.Close(); // give tester a chance to modify app state in the store ModifyAppState(); // load state store = stateStoreHelper.GetRMStateStore(); store.SetRMDispatcher(dispatcher); RMStateStore.RMState state = store.LoadState(); IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState (); ApplicationStateData appState = rmAppState[appId1]; // app is loaded NUnit.Framework.Assert.IsNotNull(appState); // app is loaded correctly NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime()); NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime()); // submission context is loaded correctly NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext( ).GetApplicationId()); ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1); // attempt1 is loaded correctly NUnit.Framework.Assert.IsNotNull(attemptState); NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId()); NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus()); // attempt1 container is loaded correctly NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId ()); // attempt1 client token master key is loaded correctly Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName)); attemptState = appState.GetAttempt(attemptId2); // attempt2 is loaded correctly NUnit.Framework.Assert.IsNotNull(attemptState); NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId()); // attempt2 container is loaded correctly NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId ()); // attempt2 client token master key is loaded correctly Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName)); //******* update application/attempt state *******// ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime (), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext (), RMAppState.Finished, "appDiagnostics", 1234); appState2.attempts.PutAll(appState.attempts); store.UpdateApplicationState(appState2); ApplicationAttemptStateData oldAttemptState = attemptState; ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance (oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState .GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished , "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100, oldAttemptState.GetFinishTime(), 0, 0); store.UpdateApplicationAttemptState(newAttemptState); // test updating the state of an app/attempt whose initial state was not // saved. ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10); ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl (); dummyContext.SetApplicationId(dummyAppId); ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime (), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished , "appDiagnostics", 1234); store.UpdateApplicationState(dummyApp); ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId , 6); ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance (dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens (), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime (), 0, 0); store.UpdateApplicationAttemptState(dummyAttempt); // let things settle down Sharpen.Thread.Sleep(1000); store.Close(); // check updated application state. store = stateStoreHelper.GetRMStateStore(); store.SetRMDispatcher(dispatcher); RMStateStore.RMState newRMState = store.LoadState(); IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState (); NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext ().GetApplicationId()]); ApplicationStateData updatedAppState = newRMAppState[appId1]; NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId (), updatedAppState.GetApplicationSubmissionContext().GetApplicationId()); NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime ()); NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime ()); NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser()); // new app state fields NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState()); NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics( )); NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime()); // check updated attempt state NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext ().GetApplicationId()].GetAttempt(dummyAttemptId)); ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState .GetAttemptId()); NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState .GetAttemptId()); NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer ().GetId()); Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName)); // new attempt state fields NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState ()); NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl ()); NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics ()); NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus ()); NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState .GetFinalApplicationStatus()); // assert store is in expected state after everything is cleaned NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid()); store.Close(); }
/// <exception cref="System.Exception"/> protected internal virtual ContainerId StoreAttempt(RMStateStore store, ApplicationAttemptId attemptId, string containerIdStr, Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier > appToken, SecretKey clientTokenMasterKey, RMStateStoreTestBase.TestDispatcher dispatcher) { RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics >(); Container container = new ContainerPBImpl(); container.SetId(ConverterUtils.ToContainerId(containerIdStr)); RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>(); Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId); Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container); Org.Mockito.Mockito.When(mockAttempt.GetAMRMToken()).ThenReturn(appToken); Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey ); Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics ); Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()). ThenReturn(new AggregateAppResourceUsage(0, 0)); dispatcher.attemptId = attemptId; store.StoreNewApplicationAttempt(mockAttempt); WaitNotify(dispatcher); return(container.GetId()); }