Esempio n. 1
0
        /// <exception cref="System.IO.IOException"/>
        private int LoadRMApp(RMStateStore.RMState rmState, LeveldbIterator iter, string
                              appIdStr, byte[] appData)
        {
            ApplicationStateData appState = CreateApplicationState(appIdStr, appData);
            ApplicationId        appId    = appState.GetApplicationSubmissionContext().GetApplicationId
                                                ();

            rmState.appState[appId] = appState;
            string attemptNodePrefix = GetApplicationNodeKey(appId) + Separator;

            while (iter.HasNext())
            {
                KeyValuePair <byte[], byte[]> entry = iter.PeekNext();
                string key = JniDBFactory.AsString(entry.Key);
                if (!key.StartsWith(attemptNodePrefix))
                {
                    break;
                }
                string attemptId = Sharpen.Runtime.Substring(key, attemptNodePrefix.Length);
                if (attemptId.StartsWith(ApplicationAttemptId.appAttemptIdStrPrefix))
                {
                    ApplicationAttemptStateData attemptState = CreateAttemptState(attemptId, entry.Value
                                                                                  );
                    appState.attempts[attemptState.GetAttemptId()] = attemptState;
                }
                else
                {
                    Log.Warn("Ignoring unknown application key: " + key);
                }
                iter.Next();
            }
            int numAttempts = appState.attempts.Count;

            if (Log.IsDebugEnabled())
            {
                Log.Debug("Loaded application " + appId + " with " + numAttempts + " attempts");
            }
            return(numAttempts);
        }
Esempio n. 2
0
 /// <exception cref="System.Exception"/>
 protected internal override void UpdateApplicationAttemptStateInternal(ApplicationAttemptId
                                                                        appAttemptId, ApplicationAttemptStateData attemptStateDataPB)
 {
     lock (this)
     {
         Path appDirPath     = GetAppDir(rmAppRoot, appAttemptId.GetApplicationId());
         Path nodeCreatePath = GetNodePath(appDirPath, appAttemptId.ToString());
         Log.Info("Updating info for attempt: " + appAttemptId + " at: " + nodeCreatePath);
         byte[] attemptStateData = attemptStateDataPB.GetProto().ToByteArray();
         try
         {
             // currently throw all exceptions. May need to respond differently for HA
             // based on whether we have lost the right to write to FS
             UpdateFile(nodeCreatePath, attemptStateData, true);
         }
         catch (Exception e)
         {
             Log.Info("Error updating info for attempt: " + appAttemptId, e);
             throw;
         }
     }
 }
Esempio n. 3
0
 public RMStateUpdateAppAttemptEvent(ApplicationAttemptStateData attemptState)
     : base(RMStateStoreEventType.UpdateAppAttempt)
 {
     this.attemptState = attemptState;
 }
Esempio n. 4
0
 // Do nothing
 /// <exception cref="System.Exception"/>
 protected internal override void StoreApplicationAttemptStateInternal(ApplicationAttemptId
                                                                       attemptId, ApplicationAttemptStateData attemptStateData)
 {
 }
Esempio n. 5
0
        public virtual void TestFencedState()
        {
            TestZKRMStateStore.TestZKRMStateStoreTester zkTester = new TestZKRMStateStore.TestZKRMStateStoreTester
                                                                       (this);
            RMStateStore store = zkTester.GetRMStateStore();

            // Move state to FENCED from ACTIVE
            store.UpdateFencedState();
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            long submitTime = Runtime.CurrentTimeMillis();
            long startTime  = submitTime + 1000;
            // Add a new app
            RMApp mockApp = Org.Mockito.Mockito.Mock <RMApp>();
            ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();

            Org.Mockito.Mockito.When(mockApp.GetSubmitTime()).ThenReturn(submitTime);
            Org.Mockito.Mockito.When(mockApp.GetStartTime()).ThenReturn(startTime);
            Org.Mockito.Mockito.When(mockApp.GetApplicationSubmissionContext()).ThenReturn(context
                                                                                           );
            Org.Mockito.Mockito.When(mockApp.GetUser()).ThenReturn("test");
            store.StoreNewApplication(mockApp);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // Add a new attempt
            ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
                                                                      ();
            ApplicationAttemptId attemptId = ConverterUtils.ToApplicationAttemptId("appattempt_1234567894321_0001_000001"
                                                                                   );
            SecretKey           clientTokenMasterKey    = clientToAMTokenMgr.CreateMasterKey(attemptId);
            RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
                                                                                    >();
            Container container = new ContainerPBImpl();

            container.SetId(ConverterUtils.ToContainerId("container_1234567891234_0001_01_000001"
                                                         ));
            RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
            Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
            Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
                                                                                       );
            Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
                                                                                      );
            Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
            ThenReturn(new AggregateAppResourceUsage(0, 0));
            store.StoreNewApplicationAttempt(mockAttempt);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            long finishTime = submitTime + 1000;
            // Update attempt
            ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
                                                              (attemptId, container, store.GetCredentialsFromAppAttempt(mockAttempt), startTime
                                                              , RMAppAttemptState.Finished, "testUrl", "test", FinalApplicationStatus.Succeeded
                                                              , 100, finishTime, 0, 0);

            store.UpdateApplicationAttemptState(newAttemptState);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // Update app
            ApplicationStateData appState = ApplicationStateData.NewInstance(submitTime, startTime
                                                                             , context, "test");

            store.UpdateApplicationState(appState);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // Remove app
            store.RemoveApplication(mockApp);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // store RM delegation token;
            RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(new Text("owner1"
                                                                                         ), new Text("renewer1"), new Text("realuser1"));
            long renewDate1 = Runtime.CurrentTimeMillis();

            dtId1.SetSequenceNumber(1111);
            store.StoreRMDelegationToken(dtId1, renewDate1);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            store.UpdateRMDelegationToken(dtId1, renewDate1);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // remove delegation key;
            store.RemoveRMDelegationToken(dtId1);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // store delegation master key;
            DelegationKey key = new DelegationKey(1234, 4321, Sharpen.Runtime.GetBytesForString
                                                      ("keyBytes"));

            store.StoreRMDTMasterKey(key);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // remove delegation master key;
            store.RemoveRMDTMasterKey(key);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // store or update AMRMToken;
            store.StoreOrUpdateAMRMTokenSecretManager(null, false);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            store.Close();
        }
        /// <exception cref="System.Exception"/>
        internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper
                                                  stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier)
        {
            long          submitTime = Runtime.CurrentTimeMillis();
            long          startTime  = Runtime.CurrentTimeMillis() + 1234;
            Configuration conf       = new YarnConfiguration();
            RMStateStore  store      = stateStoreHelper.GetRMStateStore();

            RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
                                                                 ();
            store.SetRMDispatcher(dispatcher);
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
            AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager
                                                                             (conf, rmContext));
            MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey();

            Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData);
            ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
                                                                      ();
            ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001"
                                                                                    );
            ApplicationId appId1 = attemptId1.GetApplicationId();

            StoreApp(store, appId1, submitTime, startTime);
            verifier.AfterStoreApp(store, appId1);
            // create application token and client token key for attempt1
            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken
                                                                                                (attemptId1, appTokenMgr);
            SecretKey   clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1);
            ContainerId containerId1    = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001"
                                                       , appAttemptToken1, clientTokenKey1, dispatcher);
            string appAttemptIdStr2         = "appattempt_1352994193343_0001_000002";
            ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2
                                                                                    );

            // create application token and client token key for attempt2
            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken
                                                                                                (attemptId2, appTokenMgr);
            SecretKey   clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2);
            ContainerId containerId2    = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001"
                                                       , appAttemptToken2, clientTokenKey2, dispatcher);
            ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
                                                                                          );
            ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();

            StoreApp(store, appIdRemoved, submitTime, startTime);
            StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
                         , null, dispatcher);
            verifier.AfterStoreAppAttempt(store, attemptIdRemoved);
            RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>();
            RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
                                                                                    >();
            Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId
                                                                                       , RMAppAttempt>();
            ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();

            context.SetApplicationId(appIdRemoved);
            Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime);
            Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn
                (context);
            Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts);
            Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1");
            RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved
                                                                                      );
            Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn(
                mockRmAppAttemptMetrics);
            Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
            ThenReturn(new AggregateAppResourceUsage(0, 0));
            attempts[attemptIdRemoved] = mockRemovedAttempt;
            store.RemoveApplication(mockRemovedApp);
            // remove application directory recursively.
            StoreApp(store, appIdRemoved, submitTime, startTime);
            StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
                         , null, dispatcher);
            store.RemoveApplication(mockRemovedApp);
            // let things settle down
            Sharpen.Thread.Sleep(1000);
            store.Close();
            // give tester a chance to modify app state in the store
            ModifyAppState();
            // load state
            store = stateStoreHelper.GetRMStateStore();
            store.SetRMDispatcher(dispatcher);
            RMStateStore.RMState state = store.LoadState();
            IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState
                                                                               ();
            ApplicationStateData appState = rmAppState[appId1];

            // app is loaded
            NUnit.Framework.Assert.IsNotNull(appState);
            // app is loaded correctly
            NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime());
            NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime());
            // submission context is loaded correctly
            NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext(
                                                ).GetApplicationId());
            ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1);

            // attempt1 is loaded correctly
            NUnit.Framework.Assert.IsNotNull(attemptState);
            NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId());
            NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus());
            // attempt1 container is loaded correctly
            NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId
                                                ());
            // attempt1 client token master key is loaded correctly
            Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens
                                         ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
            attemptState = appState.GetAttempt(attemptId2);
            // attempt2 is loaded correctly
            NUnit.Framework.Assert.IsNotNull(attemptState);
            NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId());
            // attempt2 container is loaded correctly
            NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId
                                                ());
            // attempt2 client token master key is loaded correctly
            Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
                                         ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
            //******* update application/attempt state *******//
            ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime
                                                                                  (), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext
                                                                                  (), RMAppState.Finished, "appDiagnostics", 1234);

            appState2.attempts.PutAll(appState.attempts);
            store.UpdateApplicationState(appState2);
            ApplicationAttemptStateData oldAttemptState = attemptState;
            ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
                                                              (oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState
                                                              .GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished
                                                              , "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100,
                                                              oldAttemptState.GetFinishTime(), 0, 0);

            store.UpdateApplicationAttemptState(newAttemptState);
            // test updating the state of an app/attempt whose initial state was not
            // saved.
            ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10);
            ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl
                                                            ();

            dummyContext.SetApplicationId(dummyAppId);
            ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime
                                                                                 (), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished
                                                                             , "appDiagnostics", 1234);

            store.UpdateApplicationState(dummyApp);
            ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId
                                                                                   , 6);
            ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance
                                                           (dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens
                                                               (), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl",
                                                           "attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime
                                                               (), 0, 0);

            store.UpdateApplicationAttemptState(dummyAttempt);
            // let things settle down
            Sharpen.Thread.Sleep(1000);
            store.Close();
            // check updated application state.
            store = stateStoreHelper.GetRMStateStore();
            store.SetRMDispatcher(dispatcher);
            RMStateStore.RMState newRMState = store.LoadState();
            IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState
                                                                                  ();

            NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
                                                               ().GetApplicationId()]);
            ApplicationStateData updatedAppState = newRMAppState[appId1];

            NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId
                                                (), updatedAppState.GetApplicationSubmissionContext().GetApplicationId());
            NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime
                                                ());
            NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime
                                                ());
            NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser());
            // new app state fields
            NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState());
            NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics(
                                                ));
            NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime());
            // check updated attempt state
            NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
                                                               ().GetApplicationId()].GetAttempt(dummyAttemptId));
            ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState
                                                                                         .GetAttemptId());

            NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState
                                            .GetAttemptId());
            NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer
                                                ().GetId());
            Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
                                         ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
            // new attempt state fields
            NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState
                                                ());
            NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl
                                                ());
            NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics
                                                ());
            NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus
                                                ());
            NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState
                                            .GetFinalApplicationStatus());
            // assert store is in expected state after everything is cleaned
            NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid());
            store.Close();
        }
Esempio n. 7
0
 /// <exception cref="System.Exception"/>
 protected internal override void UpdateApplicationAttemptStateInternal(ApplicationAttemptId
                                                                        appAttemptId, ApplicationAttemptStateData attemptState)
 {
     lock (this)
     {
         ApplicationStateData appState = state.GetApplicationState()[appAttemptId.GetApplicationId
                                                                         ()];
         if (appState == null)
         {
             throw new YarnRuntimeException("Application doesn't exist");
         }
         Log.Info("Updating final state " + attemptState.GetState() + " for attempt: " + attemptState
                  .GetAttemptId());
         appState.attempts[attemptState.GetAttemptId()] = attemptState;
     }
 }
Esempio n. 8
0
 /// <exception cref="System.Exception"/>
 protected internal override void StoreApplicationAttemptStateInternal(ApplicationAttemptId
                                                                       appAttemptId, ApplicationAttemptStateData attemptState)
 {
     lock (this)
     {
         ApplicationStateData appState = state.GetApplicationState()[attemptState.GetAttemptId
                                                                         ().GetApplicationId()];
         if (appState == null)
         {
             throw new YarnRuntimeException("Application doesn't exist");
         }
         appState.attempts[attemptState.GetAttemptId()] = attemptState;
     }
 }
Esempio n. 9
0
        /// <exception cref="System.IO.IOException"/>
        protected internal override void StoreApplicationAttemptStateInternal(ApplicationAttemptId
                                                                              attemptId, ApplicationAttemptStateData attemptStateData)
        {
            string key = GetApplicationAttemptNodeKey(attemptId);

            if (Log.IsDebugEnabled())
            {
                Log.Debug("Storing state for attempt " + attemptId + " at " + key);
            }
            try
            {
                db.Put(JniDBFactory.Bytes(key), attemptStateData.GetProto().ToByteArray());
            }
            catch (DBException e)
            {
                throw new IOException(e);
            }
        }