예제 #1
0
        public virtual void TestLogAggregationContextPassedIntoContainerToken()
        {
            MockRM rm1 = new MockRM(conf);

            rm1.Start();
            MockNM nm1 = rm1.RegisterNode("127.0.0.1:1234", 8000);
            MockNM nm2 = rm1.RegisterNode("127.0.0.1:2345", 8000);

            // LogAggregationContext is set as null
            NUnit.Framework.Assert.IsNull(GetLogAggregationContextFromContainerToken(rm1, nm1
                                                                                     , null));
            // create a not-null LogAggregationContext
            LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
                                                                                            , "excludePattern", "rolledLogsIncludePattern", "rolledLogsExcludePattern");
            LogAggregationContext returned = GetLogAggregationContextFromContainerToken(rm1,
                                                                                        nm2, logAggregationContext);

            NUnit.Framework.Assert.AreEqual("includePattern", returned.GetIncludePattern());
            NUnit.Framework.Assert.AreEqual("excludePattern", returned.GetExcludePattern());
            NUnit.Framework.Assert.AreEqual("rolledLogsIncludePattern", returned.GetRolledLogsIncludePattern
                                                ());
            NUnit.Framework.Assert.AreEqual("rolledLogsExcludePattern", returned.GetRolledLogsExcludePattern
                                                ());
            rm1.Stop();
        }
        public virtual void TestApplicationRecovery()
        {
            YarnConfiguration conf = new YarnConfiguration();

            conf.SetBoolean(YarnConfiguration.NmRecoveryEnabled, true);
            conf.Set(YarnConfiguration.NmAddress, "localhost:1234");
            conf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
            conf.Set(YarnConfiguration.YarnAdminAcl, "yarn_admin_user");
            NMStateStoreService stateStore = new NMMemoryStateStoreService();

            stateStore.Init(conf);
            stateStore.Start();
            Context context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf
                                                                                          ), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore
                                                        );
            ContainerManagerImpl cm = CreateContainerManager(context);

            cm.Init(conf);
            cm.Start();
            // simulate registration with RM
            MasterKey masterKey = new MasterKeyPBImpl();

            masterKey.SetKeyId(123);
            masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 }));
            context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
            context.GetNMTokenSecretManager().SetMasterKey(masterKey);
            // add an application by starting a container
            string               appUser   = "******";
            string               modUser   = "******";
            string               viewUser  = "******";
            string               enemyUser = "******";
            ApplicationId        appId     = ApplicationId.NewInstance(0, 1);
            ApplicationAttemptId attemptId = ApplicationAttemptId.NewInstance(appId, 1);
            ContainerId          cid       = ContainerId.NewContainerId(attemptId, 1);
            IDictionary <string, LocalResource> localResources = Collections.EmptyMap();
            IDictionary <string, string>        containerEnv   = Sharpen.Collections.EmptyMap();
            IList <string> containerCmds = Sharpen.Collections.EmptyList();
            IDictionary <string, ByteBuffer> serviceData = Sharpen.Collections.EmptyMap();
            Credentials      containerCreds = new Credentials();
            DataOutputBuffer dob            = new DataOutputBuffer();

            containerCreds.WriteTokenStorageToStream(dob);
            ByteBuffer containerTokens = ByteBuffer.Wrap(dob.GetData(), 0, dob.GetLength());
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>();

            acls[ApplicationAccessType.ModifyApp] = modUser;
            acls[ApplicationAccessType.ViewApp]   = viewUser;
            ContainerLaunchContext clc = ContainerLaunchContext.NewInstance(localResources, containerEnv
                                                                            , containerCmds, serviceData, containerTokens, acls);
            // create the logAggregationContext
            LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
                                                                                            , "excludePattern", "includePatternInRollingAggregation", "excludePatternInRollingAggregation"
                                                                                            );
            StartContainersResponse startResponse = StartContainer(context, cm, cid, clc, logAggregationContext
                                                                   );

            NUnit.Framework.Assert.IsTrue(startResponse.GetFailedRequests().IsEmpty());
            NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = context.GetApplications()[appId];
            NUnit.Framework.Assert.IsNotNull(app);
            WaitForAppState(app, ApplicationState.Initing);
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
            // reset container manager and verify app recovered with proper acls
            cm.Stop();
            context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
                                                NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
            cm = CreateContainerManager(context);
            cm.Init(conf);
            cm.Start();
            NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
            app = context.GetApplications()[appId];
            NUnit.Framework.Assert.IsNotNull(app);
            // check whether LogAggregationContext is recovered correctly
            LogAggregationContext recovered = ((ApplicationImpl)app).GetLogAggregationContext
                                                  ();

            NUnit.Framework.Assert.IsNotNull(recovered);
            NUnit.Framework.Assert.AreEqual(logAggregationContext.GetIncludePattern(), recovered
                                            .GetIncludePattern());
            NUnit.Framework.Assert.AreEqual(logAggregationContext.GetExcludePattern(), recovered
                                            .GetExcludePattern());
            NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsIncludePattern
                                                (), recovered.GetRolledLogsIncludePattern());
            NUnit.Framework.Assert.AreEqual(logAggregationContext.GetRolledLogsExcludePattern
                                                (), recovered.GetRolledLogsExcludePattern());
            WaitForAppState(app, ApplicationState.Initing);
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
            // simulate application completion
            IList <ApplicationId> finishedApps = new AList <ApplicationId>();

            finishedApps.AddItem(appId);
            cm.Handle(new CMgrCompletedAppsEvent(finishedApps, CMgrCompletedAppsEvent.Reason.
                                                 ByResourcemanager));
            WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
            // restart and verify app is marked for finishing
            cm.Stop();
            context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
                                                NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
            cm = CreateContainerManager(context);
            cm.Init(conf);
            cm.Start();
            NUnit.Framework.Assert.AreEqual(1, context.GetApplications().Count);
            app = context.GetApplications()[appId];
            NUnit.Framework.Assert.IsNotNull(app);
            WaitForAppState(app, ApplicationState.ApplicationResourcesCleaningup);
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(modUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(viewUser), ApplicationAccessType.ModifyApp, appUser, appId));
            NUnit.Framework.Assert.IsTrue(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                          .CreateRemoteUser(viewUser), ApplicationAccessType.ViewApp, appUser, appId));
            NUnit.Framework.Assert.IsFalse(context.GetApplicationACLsManager().CheckAccess(UserGroupInformation
                                                                                           .CreateRemoteUser(enemyUser), ApplicationAccessType.ViewApp, appUser, appId));
            // simulate log aggregation completion
            app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationResourcesCleanedup
                                            ));
            NUnit.Framework.Assert.AreEqual(app.GetApplicationState(), ApplicationState.Finished
                                            );
            app.Handle(new ApplicationEvent(app.GetAppId(), ApplicationEventType.ApplicationLogHandlingFinished
                                            ));
            // restart and verify app is no longer present after recovery
            cm.Stop();
            context = new NodeManager.NMContext(new NMContainerTokenSecretManager(conf), new
                                                NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore);
            cm = CreateContainerManager(context);
            cm.Init(conf);
            cm.Start();
            NUnit.Framework.Assert.IsTrue(context.GetApplications().IsEmpty());
            cm.Stop();
        }