コード例 #1
0
        private static RMApp CreateRMApp(ApplicationId appId)
        {
            RMApp app = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
            Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L);
            Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L);
            Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L);
            Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
                                                                                        ));
            RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId
                                                                              .NewInstance(appId, 1));
            Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt);
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Undefined);
            Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null,
                                                                                        0, 0, int.MaxValue, long.MaxValue));
            return(app);
        }
コード例 #2
0
        public virtual void TestAMLaunchAndCleanup()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            TestApplicationMasterLauncher.MyContainerManagerImpl containerManager = new TestApplicationMasterLauncher.MyContainerManagerImpl
                                                                                        ();
            MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(containerManager);

            rm.Start();
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
            RMApp  app = rm.SubmitApp(2000);

            // kick the scheduling
            nm1.NodeHeartbeat(true);
            int waitCount = 0;

            while (containerManager.launched == false && waitCount++ < 20)
            {
                Log.Info("Waiting for AM Launch to happen..");
                Sharpen.Thread.Sleep(1000);
            }
            NUnit.Framework.Assert.IsTrue(containerManager.launched);
            RMAppAttempt         attempt      = app.GetCurrentAppAttempt();
            ApplicationAttemptId appAttemptId = attempt.GetAppAttemptId();

            NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), containerManager.attemptIdAtContainerManager
                                            );
            NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), containerManager.submitTimeAtContainerManager
                                            );
            NUnit.Framework.Assert.AreEqual(app.GetRMAppAttempt(appAttemptId).GetMasterContainer
                                                ().GetId().ToString(), containerManager.containerIdAtContainerManager);
            NUnit.Framework.Assert.AreEqual(nm1.GetNodeId().ToString(), containerManager.nmHostAtContainerManager
                                            );
            NUnit.Framework.Assert.AreEqual(YarnConfiguration.DefaultRmAmMaxAttempts, containerManager
                                            .maxAppAttempts);
            MockAM am = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), appAttemptId
                                   );

            am.RegisterAppAttempt();
            am.UnregisterAppAttempt();
            //complete the AM container to finish the app normally
            nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
            am.WaitForState(RMAppAttemptState.Finished);
            waitCount = 0;
            while (containerManager.cleanedup == false && waitCount++ < 20)
            {
                Log.Info("Waiting for AM Cleanup to happen..");
                Sharpen.Thread.Sleep(1000);
            }
            NUnit.Framework.Assert.IsTrue(containerManager.cleanedup);
            am.WaitForState(RMAppAttemptState.Finished);
            rm.Stop();
        }
コード例 #3
0
        /// <exception cref="System.Exception"/>
        protected internal virtual RMApp StoreApp(RMStateStore store, ApplicationId appId
                                                  , long submitTime, long startTime)
        {
            ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();

            context.SetApplicationId(appId);
            RMApp mockApp = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(mockApp.GetApplicationId()).ThenReturn(appId);
            Org.Mockito.Mockito.When(mockApp.GetSubmitTime()).ThenReturn(submitTime);
            Org.Mockito.Mockito.When(mockApp.GetStartTime()).ThenReturn(startTime);
            Org.Mockito.Mockito.When(mockApp.GetApplicationSubmissionContext()).ThenReturn(context
                                                                                           );
            Org.Mockito.Mockito.When(mockApp.GetUser()).ThenReturn("test");
            store.StoreNewApplication(mockApp);
            return(mockApp);
        }
コード例 #4
0
        private static RMApp CreateRMApp(ApplicationId appId)
        {
            RMApp app = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
            Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
            Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
            Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
            Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(0L);
            Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(1L);
            Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(2L);
            Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
                                                                                        ));
            Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
                                                                                 .Undefined);
            return(app);
        }
コード例 #5
0
 public virtual void AppCreated(RMApp app, long createdTime)
 {
     if (publishSystemMetrics)
     {
         dispatcher.GetEventHandler().Handle(new ApplicationCreatedEvent(app.GetApplicationId
                                                                             (), app.GetName(), app.GetApplicationType(), app.GetUser(), app.GetQueue(), app.
                                                                         GetSubmitTime(), createdTime));
     }
 }
コード例 #6
0
        public virtual int Run(string[] args)
        {
            Log.Info("Starting ZKRMStateStorePerf ver." + version);
            int    numApp = ZkPerfNumAppDefault;
            int    numAppAttemptPerApp = ZkPerfNumAppattemptPerApp;
            string hostPort            = null;
            bool   launchLocalZK       = true;

            if (args.Length == 0)
            {
                System.Console.Error.WriteLine("Missing arguments.");
                return(-1);
            }
            for (int i = 0; i < args.Length; i++)
            {
                // parse command line
                if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-appsize"))
                {
                    numApp = System.Convert.ToInt32(args[++i]);
                }
                else
                {
                    if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-appattemptsize"))
                    {
                        numAppAttemptPerApp = System.Convert.ToInt32(args[++i]);
                    }
                    else
                    {
                        if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-hostPort"))
                        {
                            hostPort      = args[++i];
                            launchLocalZK = false;
                        }
                        else
                        {
                            if (Sharpen.Runtime.EqualsIgnoreCase(args[i], "-workingZnode"))
                            {
                                workingZnode = args[++i];
                            }
                            else
                            {
                                System.Console.Error.WriteLine("Illegal argument: " + args[i]);
                                return(-1);
                            }
                        }
                    }
                }
            }
            if (launchLocalZK)
            {
                try
                {
                    SetUp();
                }
                catch (Exception e)
                {
                    System.Console.Error.WriteLine("failed to setup. : " + e.Message);
                    return(-1);
                }
            }
            InitStore(hostPort);
            long submitTime = Runtime.CurrentTimeMillis();
            long startTime  = Runtime.CurrentTimeMillis() + 1234;
            AList <ApplicationId>        applicationIds = new AList <ApplicationId>();
            AList <RMApp>                rmApps         = new AList <RMApp>();
            AList <ApplicationAttemptId> attemptIds     = new AList <ApplicationAttemptId>();
            Dictionary <ApplicationId, ICollection <ApplicationAttemptId> > appIdsToAttemptId =
                new Dictionary <ApplicationId, ICollection <ApplicationAttemptId> >();

            RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
                                                                 ();
            store.SetRMDispatcher(dispatcher);
            for (int i_1 = 0; i_1 < numApp; i_1++)
            {
                ApplicationId appId = ApplicationId.NewInstance(clusterTimeStamp, i_1);
                applicationIds.AddItem(appId);
                AList <ApplicationAttemptId> attemptIdsForThisApp = new AList <ApplicationAttemptId
                                                                               >();
                for (int j = 0; j < numAppAttemptPerApp; j++)
                {
                    ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, j);
                    attemptIdsForThisApp.AddItem(attemptId);
                }
                appIdsToAttemptId[appId] = new LinkedHashSet(attemptIdsForThisApp);
                Sharpen.Collections.AddAll(attemptIds, attemptIdsForThisApp);
            }
            foreach (ApplicationId appId_1 in applicationIds)
            {
                RMApp app = null;
                try
                {
                    app = StoreApp(store, appId_1, submitTime, startTime);
                }
                catch (Exception e)
                {
                    System.Console.Error.WriteLine("failed to create Application Znode. : " + e.Message
                                                   );
                    return(-1);
                }
                WaitNotify(dispatcher);
                rmApps.AddItem(app);
            }
            foreach (ApplicationAttemptId attemptId_1 in attemptIds)
            {
                Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> tokenId = GenerateAMRMToken
                                                                                           (attemptId_1, appTokenMgr);
                SecretKey clientTokenKey = clientToAMTokenMgr.CreateMasterKey(attemptId_1);
                try
                {
                    StoreAttempt(store, attemptId_1, ContainerId.NewContainerId(attemptId_1, 0L).ToString
                                     (), tokenId, clientTokenKey, dispatcher);
                }
                catch (Exception e)
                {
                    System.Console.Error.WriteLine("failed to create AppAttempt Znode. : " + e.Message
                                                   );
                    return(-1);
                }
            }
            long storeStart = Runtime.CurrentTimeMillis();

            try
            {
                store.LoadState();
            }
            catch (Exception e)
            {
                System.Console.Error.WriteLine("failed to locaState from ZKRMStateStore. : " + e.
                                               Message);
                return(-1);
            }
            long   storeEnd  = Runtime.CurrentTimeMillis();
            long   loadTime  = storeEnd - storeStart;
            string resultMsg = "ZKRMStateStore takes " + loadTime + " msec to loadState.";

            Log.Info(resultMsg);
            System.Console.Out.WriteLine(resultMsg);
            // cleanup
            try
            {
                foreach (RMApp app in rmApps)
                {
                    ApplicationStateData appState = ApplicationStateData.NewInstance(app.GetSubmitTime
                                                                                         (), app.GetStartTime(), app.GetApplicationSubmissionContext(), app.GetUser());
                    ApplicationId appId = app.GetApplicationId();
                    IDictionary   m     = Org.Mockito.Mockito.Mock <IDictionary>();
                    Org.Mockito.Mockito.When(m.Keys).ThenReturn(appIdsToAttemptId[appId_1]);
                    appState.attempts = m;
                    store.RemoveApplicationStateInternal(appState);
                }
            }
            catch (Exception e)
            {
                System.Console.Error.WriteLine("failed to cleanup. : " + e.Message);
                return(-1);
            }
            return(0);
        }
コード例 #7
0
        public virtual void TestFencedState()
        {
            TestZKRMStateStore.TestZKRMStateStoreTester zkTester = new TestZKRMStateStore.TestZKRMStateStoreTester
                                                                       (this);
            RMStateStore store = zkTester.GetRMStateStore();

            // Move state to FENCED from ACTIVE
            store.UpdateFencedState();
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            long submitTime = Runtime.CurrentTimeMillis();
            long startTime  = submitTime + 1000;
            // Add a new app
            RMApp mockApp = Org.Mockito.Mockito.Mock <RMApp>();
            ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();

            Org.Mockito.Mockito.When(mockApp.GetSubmitTime()).ThenReturn(submitTime);
            Org.Mockito.Mockito.When(mockApp.GetStartTime()).ThenReturn(startTime);
            Org.Mockito.Mockito.When(mockApp.GetApplicationSubmissionContext()).ThenReturn(context
                                                                                           );
            Org.Mockito.Mockito.When(mockApp.GetUser()).ThenReturn("test");
            store.StoreNewApplication(mockApp);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // Add a new attempt
            ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
                                                                      ();
            ApplicationAttemptId attemptId = ConverterUtils.ToApplicationAttemptId("appattempt_1234567894321_0001_000001"
                                                                                   );
            SecretKey           clientTokenMasterKey    = clientToAMTokenMgr.CreateMasterKey(attemptId);
            RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
                                                                                    >();
            Container container = new ContainerPBImpl();

            container.SetId(ConverterUtils.ToContainerId("container_1234567891234_0001_01_000001"
                                                         ));
            RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
            Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
            Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
                                                                                       );
            Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
                                                                                      );
            Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
            ThenReturn(new AggregateAppResourceUsage(0, 0));
            store.StoreNewApplicationAttempt(mockAttempt);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            long finishTime = submitTime + 1000;
            // Update attempt
            ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
                                                              (attemptId, container, store.GetCredentialsFromAppAttempt(mockAttempt), startTime
                                                              , RMAppAttemptState.Finished, "testUrl", "test", FinalApplicationStatus.Succeeded
                                                              , 100, finishTime, 0, 0);

            store.UpdateApplicationAttemptState(newAttemptState);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // Update app
            ApplicationStateData appState = ApplicationStateData.NewInstance(submitTime, startTime
                                                                             , context, "test");

            store.UpdateApplicationState(appState);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // Remove app
            store.RemoveApplication(mockApp);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // store RM delegation token;
            RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(new Text("owner1"
                                                                                         ), new Text("renewer1"), new Text("realuser1"));
            long renewDate1 = Runtime.CurrentTimeMillis();

            dtId1.SetSequenceNumber(1111);
            store.StoreRMDelegationToken(dtId1, renewDate1);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            store.UpdateRMDelegationToken(dtId1, renewDate1);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // remove delegation key;
            store.RemoveRMDelegationToken(dtId1);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // store delegation master key;
            DelegationKey key = new DelegationKey(1234, 4321, Sharpen.Runtime.GetBytesForString
                                                      ("keyBytes"));

            store.StoreRMDTMasterKey(key);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // remove delegation master key;
            store.RemoveRMDTMasterKey(key);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            // store or update AMRMToken;
            store.StoreOrUpdateAMRMTokenSecretManager(null, false);
            NUnit.Framework.Assert.AreEqual("RMStateStore should have been in fenced state",
                                            true, store.IsFencedState());
            store.Close();
        }
コード例 #8
0
 public virtual void ApplicationStarted(RMApp app)
 {
     if (historyServiceEnabled)
     {
         dispatcher.GetEventHandler().Handle(new WritingApplicationStartEvent(app.GetApplicationId
                                                                                  (), ApplicationStartData.NewInstance(app.GetApplicationId(), app.GetName(), app.
                                                                                                                       GetApplicationType(), app.GetQueue(), app.GetUser(), app.GetSubmitTime(), app.GetStartTime
                                                                                                                           ())));
     }
 }
コード例 #9
0
        /// <exception cref="System.Exception"/>
        internal virtual void TestRMAppStateStore(RMStateStoreTestBase.RMStateStoreHelper
                                                  stateStoreHelper, RMStateStoreTestBase.StoreStateVerifier verifier)
        {
            long          submitTime = Runtime.CurrentTimeMillis();
            long          startTime  = Runtime.CurrentTimeMillis() + 1234;
            Configuration conf       = new YarnConfiguration();
            RMStateStore  store      = stateStoreHelper.GetRMStateStore();

            RMStateStoreTestBase.TestDispatcher dispatcher = new RMStateStoreTestBase.TestDispatcher
                                                                 ();
            store.SetRMDispatcher(dispatcher);
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetStateStore()).ThenReturn(store);
            AMRMTokenSecretManager appTokenMgr = Org.Mockito.Mockito.Spy(new AMRMTokenSecretManager
                                                                             (conf, rmContext));
            MasterKeyData masterKeyData = appTokenMgr.CreateNewMasterKey();

            Org.Mockito.Mockito.When(appTokenMgr.GetMasterKey()).ThenReturn(masterKeyData);
            ClientToAMTokenSecretManagerInRM clientToAMTokenMgr = new ClientToAMTokenSecretManagerInRM
                                                                      ();
            ApplicationAttemptId attemptId1 = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0001_000001"
                                                                                    );
            ApplicationId appId1 = attemptId1.GetApplicationId();

            StoreApp(store, appId1, submitTime, startTime);
            verifier.AfterStoreApp(store, appId1);
            // create application token and client token key for attempt1
            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken1 = GenerateAMRMToken
                                                                                                (attemptId1, appTokenMgr);
            SecretKey   clientTokenKey1 = clientToAMTokenMgr.CreateMasterKey(attemptId1);
            ContainerId containerId1    = StoreAttempt(store, attemptId1, "container_1352994193343_0001_01_000001"
                                                       , appAttemptToken1, clientTokenKey1, dispatcher);
            string appAttemptIdStr2         = "appattempt_1352994193343_0001_000002";
            ApplicationAttemptId attemptId2 = ConverterUtils.ToApplicationAttemptId(appAttemptIdStr2
                                                                                    );

            // create application token and client token key for attempt2
            Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> appAttemptToken2 = GenerateAMRMToken
                                                                                                (attemptId2, appTokenMgr);
            SecretKey   clientTokenKey2 = clientToAMTokenMgr.CreateMasterKey(attemptId2);
            ContainerId containerId2    = StoreAttempt(store, attemptId2, "container_1352994193343_0001_02_000001"
                                                       , appAttemptToken2, clientTokenKey2, dispatcher);
            ApplicationAttemptId attemptIdRemoved = ConverterUtils.ToApplicationAttemptId("appattempt_1352994193343_0002_000001"
                                                                                          );
            ApplicationId appIdRemoved = attemptIdRemoved.GetApplicationId();

            StoreApp(store, appIdRemoved, submitTime, startTime);
            StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
                         , null, dispatcher);
            verifier.AfterStoreAppAttempt(store, attemptIdRemoved);
            RMApp mockRemovedApp = Org.Mockito.Mockito.Mock <RMApp>();
            RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
                                                                                    >();
            Dictionary <ApplicationAttemptId, RMAppAttempt> attempts = new Dictionary <ApplicationAttemptId
                                                                                       , RMAppAttempt>();
            ApplicationSubmissionContext context = new ApplicationSubmissionContextPBImpl();

            context.SetApplicationId(appIdRemoved);
            Org.Mockito.Mockito.When(mockRemovedApp.GetSubmitTime()).ThenReturn(submitTime);
            Org.Mockito.Mockito.When(mockRemovedApp.GetApplicationSubmissionContext()).ThenReturn
                (context);
            Org.Mockito.Mockito.When(mockRemovedApp.GetAppAttempts()).ThenReturn(attempts);
            Org.Mockito.Mockito.When(mockRemovedApp.GetUser()).ThenReturn("user1");
            RMAppAttempt mockRemovedAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();

            Org.Mockito.Mockito.When(mockRemovedAttempt.GetAppAttemptId()).ThenReturn(attemptIdRemoved
                                                                                      );
            Org.Mockito.Mockito.When(mockRemovedAttempt.GetRMAppAttemptMetrics()).ThenReturn(
                mockRmAppAttemptMetrics);
            Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
            ThenReturn(new AggregateAppResourceUsage(0, 0));
            attempts[attemptIdRemoved] = mockRemovedAttempt;
            store.RemoveApplication(mockRemovedApp);
            // remove application directory recursively.
            StoreApp(store, appIdRemoved, submitTime, startTime);
            StoreAttempt(store, attemptIdRemoved, "container_1352994193343_0002_01_000001", null
                         , null, dispatcher);
            store.RemoveApplication(mockRemovedApp);
            // let things settle down
            Sharpen.Thread.Sleep(1000);
            store.Close();
            // give tester a chance to modify app state in the store
            ModifyAppState();
            // load state
            store = stateStoreHelper.GetRMStateStore();
            store.SetRMDispatcher(dispatcher);
            RMStateStore.RMState state = store.LoadState();
            IDictionary <ApplicationId, ApplicationStateData> rmAppState = state.GetApplicationState
                                                                               ();
            ApplicationStateData appState = rmAppState[appId1];

            // app is loaded
            NUnit.Framework.Assert.IsNotNull(appState);
            // app is loaded correctly
            NUnit.Framework.Assert.AreEqual(submitTime, appState.GetSubmitTime());
            NUnit.Framework.Assert.AreEqual(startTime, appState.GetStartTime());
            // submission context is loaded correctly
            NUnit.Framework.Assert.AreEqual(appId1, appState.GetApplicationSubmissionContext(
                                                ).GetApplicationId());
            ApplicationAttemptStateData attemptState = appState.GetAttempt(attemptId1);

            // attempt1 is loaded correctly
            NUnit.Framework.Assert.IsNotNull(attemptState);
            NUnit.Framework.Assert.AreEqual(attemptId1, attemptState.GetAttemptId());
            NUnit.Framework.Assert.AreEqual(-1000, attemptState.GetAMContainerExitStatus());
            // attempt1 container is loaded correctly
            NUnit.Framework.Assert.AreEqual(containerId1, attemptState.GetMasterContainer().GetId
                                                ());
            // attempt1 client token master key is loaded correctly
            Assert.AssertArrayEquals(clientTokenKey1.GetEncoded(), attemptState.GetAppAttemptTokens
                                         ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
            attemptState = appState.GetAttempt(attemptId2);
            // attempt2 is loaded correctly
            NUnit.Framework.Assert.IsNotNull(attemptState);
            NUnit.Framework.Assert.AreEqual(attemptId2, attemptState.GetAttemptId());
            // attempt2 container is loaded correctly
            NUnit.Framework.Assert.AreEqual(containerId2, attemptState.GetMasterContainer().GetId
                                                ());
            // attempt2 client token master key is loaded correctly
            Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
                                         ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
            //******* update application/attempt state *******//
            ApplicationStateData appState2 = ApplicationStateData.NewInstance(appState.GetSubmitTime
                                                                                  (), appState.GetStartTime(), appState.GetUser(), appState.GetApplicationSubmissionContext
                                                                                  (), RMAppState.Finished, "appDiagnostics", 1234);

            appState2.attempts.PutAll(appState.attempts);
            store.UpdateApplicationState(appState2);
            ApplicationAttemptStateData oldAttemptState = attemptState;
            ApplicationAttemptStateData newAttemptState = ApplicationAttemptStateData.NewInstance
                                                              (oldAttemptState.GetAttemptId(), oldAttemptState.GetMasterContainer(), oldAttemptState
                                                              .GetAppAttemptTokens(), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished
                                                              , "myTrackingUrl", "attemptDiagnostics", FinalApplicationStatus.Succeeded, 100,
                                                              oldAttemptState.GetFinishTime(), 0, 0);

            store.UpdateApplicationAttemptState(newAttemptState);
            // test updating the state of an app/attempt whose initial state was not
            // saved.
            ApplicationId dummyAppId = ApplicationId.NewInstance(1234, 10);
            ApplicationSubmissionContext dummyContext = new ApplicationSubmissionContextPBImpl
                                                            ();

            dummyContext.SetApplicationId(dummyAppId);
            ApplicationStateData dummyApp = ApplicationStateData.NewInstance(appState.GetSubmitTime
                                                                                 (), appState.GetStartTime(), appState.GetUser(), dummyContext, RMAppState.Finished
                                                                             , "appDiagnostics", 1234);

            store.UpdateApplicationState(dummyApp);
            ApplicationAttemptId dummyAttemptId = ApplicationAttemptId.NewInstance(dummyAppId
                                                                                   , 6);
            ApplicationAttemptStateData dummyAttempt = ApplicationAttemptStateData.NewInstance
                                                           (dummyAttemptId, oldAttemptState.GetMasterContainer(), oldAttemptState.GetAppAttemptTokens
                                                               (), oldAttemptState.GetStartTime(), RMAppAttemptState.Finished, "myTrackingUrl",
                                                           "attemptDiagnostics", FinalApplicationStatus.Succeeded, 111, oldAttemptState.GetFinishTime
                                                               (), 0, 0);

            store.UpdateApplicationAttemptState(dummyAttempt);
            // let things settle down
            Sharpen.Thread.Sleep(1000);
            store.Close();
            // check updated application state.
            store = stateStoreHelper.GetRMStateStore();
            store.SetRMDispatcher(dispatcher);
            RMStateStore.RMState newRMState = store.LoadState();
            IDictionary <ApplicationId, ApplicationStateData> newRMAppState = newRMState.GetApplicationState
                                                                                  ();

            NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
                                                               ().GetApplicationId()]);
            ApplicationStateData updatedAppState = newRMAppState[appId1];

            NUnit.Framework.Assert.AreEqual(appState.GetApplicationSubmissionContext().GetApplicationId
                                                (), updatedAppState.GetApplicationSubmissionContext().GetApplicationId());
            NUnit.Framework.Assert.AreEqual(appState.GetSubmitTime(), updatedAppState.GetSubmitTime
                                                ());
            NUnit.Framework.Assert.AreEqual(appState.GetStartTime(), updatedAppState.GetStartTime
                                                ());
            NUnit.Framework.Assert.AreEqual(appState.GetUser(), updatedAppState.GetUser());
            // new app state fields
            NUnit.Framework.Assert.AreEqual(RMAppState.Finished, updatedAppState.GetState());
            NUnit.Framework.Assert.AreEqual("appDiagnostics", updatedAppState.GetDiagnostics(
                                                ));
            NUnit.Framework.Assert.AreEqual(1234, updatedAppState.GetFinishTime());
            // check updated attempt state
            NUnit.Framework.Assert.IsNotNull(newRMAppState[dummyApp.GetApplicationSubmissionContext
                                                               ().GetApplicationId()].GetAttempt(dummyAttemptId));
            ApplicationAttemptStateData updatedAttemptState = updatedAppState.GetAttempt(newAttemptState
                                                                                         .GetAttemptId());

            NUnit.Framework.Assert.AreEqual(oldAttemptState.GetAttemptId(), updatedAttemptState
                                            .GetAttemptId());
            NUnit.Framework.Assert.AreEqual(containerId2, updatedAttemptState.GetMasterContainer
                                                ().GetId());
            Assert.AssertArrayEquals(clientTokenKey2.GetEncoded(), attemptState.GetAppAttemptTokens
                                         ().GetSecretKey(RMStateStore.AmClientTokenMasterKeyName));
            // new attempt state fields
            NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Finished, updatedAttemptState.GetState
                                                ());
            NUnit.Framework.Assert.AreEqual("myTrackingUrl", updatedAttemptState.GetFinalTrackingUrl
                                                ());
            NUnit.Framework.Assert.AreEqual("attemptDiagnostics", updatedAttemptState.GetDiagnostics
                                                ());
            NUnit.Framework.Assert.AreEqual(100, updatedAttemptState.GetAMContainerExitStatus
                                                ());
            NUnit.Framework.Assert.AreEqual(FinalApplicationStatus.Succeeded, updatedAttemptState
                                            .GetFinalApplicationStatus());
            // assert store is in expected state after everything is cleaned
            NUnit.Framework.Assert.IsTrue(stateStoreHelper.IsFinalStateValid());
            store.Close();
        }
コード例 #10
0
 /// <exception cref="System.Exception"/>
 public virtual void TestPublishApplicationMetrics()
 {
     for (int i = 1; i <= 2; ++i)
     {
         ApplicationId appId = ApplicationId.NewInstance(0, i);
         RMApp         app   = CreateRMApp(appId);
         metricsPublisher.AppCreated(app, app.GetStartTime());
         metricsPublisher.AppFinished(app, RMAppState.Finished, app.GetFinishTime());
         if (i == 1)
         {
             metricsPublisher.AppACLsUpdated(app, "uers1,user2", 4L);
         }
         else
         {
             // in case user doesn't specify the ACLs
             metricsPublisher.AppACLsUpdated(app, null, 4L);
         }
         TimelineEntity entity = null;
         do
         {
             entity = store.GetEntity(appId.ToString(), ApplicationMetricsConstants.EntityType
                                      , EnumSet.AllOf <TimelineReader.Field>());
         }while (entity == null || entity.GetEvents().Count < 3);
         // ensure three events are both published before leaving the loop
         // verify all the fields
         NUnit.Framework.Assert.AreEqual(ApplicationMetricsConstants.EntityType, entity.GetEntityType
                                             ());
         NUnit.Framework.Assert.AreEqual(app.GetApplicationId().ToString(), entity.GetEntityId
                                             ());
         NUnit.Framework.Assert.AreEqual(app.GetName(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                              .NameEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetQueue(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                               .QueueEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetUser(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                              .UserEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetApplicationType(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                         .TypeEntityInfo]);
         NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                    .SubmittedTimeEntityInfo]);
         if (i == 1)
         {
             NUnit.Framework.Assert.AreEqual("uers1,user2", entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                  .AppViewAclsEntityInfo]);
         }
         else
         {
             NUnit.Framework.Assert.AreEqual(string.Empty, entity.GetOtherInfo()[ApplicationMetricsConstants
                                                                                 .AppViewAclsEntityInfo]);
             NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetMemorySeconds(), long.Parse
                                                 (entity.GetOtherInfo()[ApplicationMetricsConstants.AppMemMetrics].ToString()));
             NUnit.Framework.Assert.AreEqual(app.GetRMAppMetrics().GetVcoreSeconds(), long.Parse
                                                 (entity.GetOtherInfo()[ApplicationMetricsConstants.AppCpuMetrics].ToString()));
         }
         bool hasCreatedEvent     = false;
         bool hasFinishedEvent    = false;
         bool hasACLsUpdatedEvent = false;
         foreach (TimelineEvent @event in entity.GetEvents())
         {
             if (@event.GetEventType().Equals(ApplicationMetricsConstants.CreatedEventType))
             {
                 hasCreatedEvent = true;
                 NUnit.Framework.Assert.AreEqual(app.GetStartTime(), @event.GetTimestamp());
             }
             else
             {
                 if (@event.GetEventType().Equals(ApplicationMetricsConstants.FinishedEventType))
                 {
                     hasFinishedEvent = true;
                     NUnit.Framework.Assert.AreEqual(app.GetFinishTime(), @event.GetTimestamp());
                     NUnit.Framework.Assert.AreEqual(app.GetDiagnostics().ToString(), @event.GetEventInfo
                                                         ()[ApplicationMetricsConstants.DiagnosticsInfoEventInfo]);
                     NUnit.Framework.Assert.AreEqual(app.GetFinalApplicationStatus().ToString(), @event
                                                     .GetEventInfo()[ApplicationMetricsConstants.FinalStatusEventInfo]);
                     NUnit.Framework.Assert.AreEqual(YarnApplicationState.Finished.ToString(), @event.
                                                     GetEventInfo()[ApplicationMetricsConstants.StateEventInfo]);
                 }
                 else
                 {
                     if (@event.GetEventType().Equals(ApplicationMetricsConstants.AclsUpdatedEventType
                                                      ))
                     {
                         hasACLsUpdatedEvent = true;
                         NUnit.Framework.Assert.AreEqual(4L, @event.GetTimestamp());
                     }
                 }
             }
         }
         NUnit.Framework.Assert.IsTrue(hasCreatedEvent && hasFinishedEvent && hasACLsUpdatedEvent
                                       );
     }
 }