Esempio n. 1
0
 /// <exception cref="System.Exception"/>
 protected internal override void StoreApplicationAttemptStateInternal(ApplicationAttemptId
                                                                       appAttemptId, ApplicationAttemptStateData attemptState)
 {
     lock (this)
     {
         ApplicationStateData appState = state.GetApplicationState()[attemptState.GetAttemptId
                                                                         ().GetApplicationId()];
         if (appState == null)
         {
             throw new YarnRuntimeException("Application doesn't exist");
         }
         appState.attempts[attemptState.GetAttemptId()] = attemptState;
     }
 }
Esempio n. 2
0
        /// <exception cref="System.Exception"/>
        public virtual void Recover(RMStateStore.RMState state)
        {
            RMStateStore store = rmContext.GetStateStore();

            System.Diagnostics.Debug.Assert(store != null);
            // recover applications
            IDictionary <ApplicationId, ApplicationStateData> appStates = state.GetApplicationState
                                                                              ();

            Log.Info("Recovering " + appStates.Count + " applications");
            foreach (ApplicationStateData appState in appStates.Values)
            {
                RecoverApplication(appState, state);
            }
        }
        /// <exception cref="System.Exception"/>
        internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper
                                                  stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier)
        {
            long          submitTime = Runtime.CurrentTimeMillis();
            long          startTime  = Runtime.CurrentTimeMillis() + 1234;
            Configuration conf       = new YarnConfiguration();
            RMStateStore  store      = stateStoreHelper.GetRMStateStore();

            RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
                                                                 ();
            store.SetRMDispatcher(dispatcher);
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
            AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager
                                                                             (conf, rmContext));
            MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey();

            Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData);
            ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
                                                                      ();
            ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001"
                                                                                    );
            ApplicationId appId1 = attemptId1.GetApplicationId();

            StoreApp(store, appId1, submitTime, startTime);
            verifier.AfterStoreApp(store, appId1);
            // create application token and client token key for attempt1
            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken
                                                                                                (attemptId1, appTokenMgr);
            SecretKey   clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1);
            ContainerId containerId1    = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001"
                                                       , appAttemptToken1, clientTokenKey1, dispatcher);
            string appAttemptIdStr2         = "appattempt_1352994193343_0001_000002";
            ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2
                                                                                    );

            // create application token and client token key for attempt2
            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken
                                                                                                (attemptId2, appTokenMgr);
            SecretKey   clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2);
            ContainerId containerId2    = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001"
                                                       , appAttemptToken2, clientTokenKey2, dispatcher);
            ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
                                                                                          );
            ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();

            StoreApp(store, appIdRemoved, submitTime, startTime);
            StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
                         , null, dispatcher);
            verifier.AfterStoreAppAttempt(store, attemptIdRemoved);
            RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>();
            RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
                                                                                    >();
            Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId
                                                                                       , RMAppAttempt>();
            ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();

            context.SetApplicationId(appIdRemoved);
            Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime);
            Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn
                (context);
            Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts);
            Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1");
            RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved
                                                                                      );
            Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn(
                mockRmAppAttemptMetrics);
            Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
            ThenReturn(new AggregateAppResourceUsage(0, 0));
            attempts[attemptIdRemoved] = mockRemovedAttempt;
            store.RemoveApplication(mockRemovedApp);
            // remove application directory recursively.
            StoreApp(store, appIdRemoved, submitTime, startTime);
            StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
                         , null, dispatcher);
            store.RemoveApplication(mockRemovedApp);
            // let things settle down
            Sharpen.Thread.Sleep(1000);
            store.Close();
            // give tester a chance to modify app state in the store
            ModifyAppState();
            // load state
            store = stateStoreHelper.GetRMStateStore();
            store.SetRMDispatcher(dispatcher);
            RMStateStore.RMState state = store.LoadState();
            IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState
                                                                               ();
            ApplicationStateData appState = rmAppState[appId1];

            // app is loaded
            NUnit.Framework.Assert.IsNotNull(appState);
            // app is loaded correctly
            NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime());
            NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime());
            // submission context is loaded correctly
            NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext(
                                                ).GetApplicationId());
            ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1);

            // attempt1 is loaded correctly
            NUnit.Framework.Assert.IsNotNull(attemptState);
            NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId());
            NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus());
            // attempt1 container is loaded correctly
            NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId
                                                ());
            // attempt1 client token master key is loaded correctly
            Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens
                                         ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
            attemptState = appState.GetAttempt(attemptId2);
            // attempt2 is loaded correctly
            NUnit.Framework.Assert.IsNotNull(attemptState);
            NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId());
            // attempt2 container is loaded correctly
            NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId
                                                ());
            // attempt2 client token master key is loaded correctly
            Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
                                         ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
            //******* update application/attempt state *******//
            ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime
                                                                                  (), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext
                                                                                  (), RMAppState.Finished, "appDiagnostics", 1234);

            appState2.attempts.PutAll(appState.attempts);
            store.UpdateApplicationState(appState2);
            ApplicationAttemptStateData oldAttemptState = attemptState;
            ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
                                                              (oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState
                                                              .GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished
                                                              , "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100,
                                                              oldAttemptState.GetFinishTime(), 0, 0);

            store.UpdateApplicationAttemptState(newAttemptState);
            // test updating the state of an app/attempt whose initial state was not
            // saved.
            ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10);
            ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl
                                                            ();

            dummyContext.SetApplicationId(dummyAppId);
            ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime
                                                                                 (), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished
                                                                             , "appDiagnostics", 1234);

            store.UpdateApplicationState(dummyApp);
            ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId
                                                                                   , 6);
            ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance
                                                           (dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens
                                                               (), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl",
                                                           "attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime
                                                               (), 0, 0);

            store.UpdateApplicationAttemptState(dummyAttempt);
            // let things settle down
            Sharpen.Thread.Sleep(1000);
            store.Close();
            // check updated application state.
            store = stateStoreHelper.GetRMStateStore();
            store.SetRMDispatcher(dispatcher);
            RMStateStore.RMState newRMState = store.LoadState();
            IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState
                                                                                  ();

            NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
                                                               ().GetApplicationId()]);
            ApplicationStateData updatedAppState = newRMAppState[appId1];

            NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId
                                                (), updatedAppState.GetApplicationSubmissionContext().GetApplicationId());
            NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime
                                                ());
            NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime
                                                ());
            NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser());
            // new app state fields
            NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState());
            NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics(
                                                ));
            NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime());
            // check updated attempt state
            NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
                                                               ().GetApplicationId()].GetAttempt(dummyAttemptId));
            ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState
                                                                                         .GetAttemptId());

            NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState
                                            .GetAttemptId());
            NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer
                                                ().GetId());
            Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
                                         ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
            // new attempt state fields
            NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState
                                                ());
            NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl
                                                ());
            NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics
                                                ());
            NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus
                                                ());
            NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState
                                            .GetFinalApplicationStatus());
            // assert store is in expected state after everything is cleaned
            NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid());
            store.Close();
        }