Esempio n. 1
0
        /// <exception cref="System.Exception"/>
        private void TestMinimumAllocation(YarnConfiguration conf, int testAlloc)
        {
            MockRM rm = new MockRM(conf);

            rm.Start();
            // Register node1
            MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
            // Submit an application
            RMApp app1 = rm.SubmitApp(testAlloc);

            // kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
            MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());

            am1.RegisterAppAttempt();
            SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId
                                                                                         ());
            int checkAlloc = conf.GetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, YarnConfiguration
                                         .DefaultRmSchedulerMinimumAllocationMb);

            NUnit.Framework.Assert.AreEqual(checkAlloc, report_nm1.GetUsedResource().GetMemory
                                                ());
            rm.Stop();
        }
Esempio n. 2
0
        public virtual void TestAdminAclsWithFileSystemBasedConfigurationProvider()
        {
            configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
                              );
            //upload default configurations
            UploadDefaultConfiguration();
            try
            {
                rm = new MockRM(configuration);
                rm.Init(configuration);
                rm.Start();
            }
            catch (Exception)
            {
                NUnit.Framework.Assert.Fail("Should not get any exceptions");
            }
            string aclStringBefore = rm.adminService.GetAccessControlList().GetAclString().Trim
                                         ();
            YarnConfiguration yarnConf = new YarnConfiguration();

            yarnConf.Set(YarnConfiguration.YarnAdminAcl, "world:anyone:rwcda");
            UploadConfiguration(yarnConf, "yarn-site.xml");
            rm.adminService.RefreshAdminAcls(RefreshAdminAclsRequest.NewInstance());
            string aclStringAfter = rm.adminService.GetAccessControlList().GetAclString().Trim
                                        ();

            NUnit.Framework.Assert.IsTrue(!aclStringAfter.Equals(aclStringBefore));
            NUnit.Framework.Assert.AreEqual(aclStringAfter, "world:anyone:rwcda," + UserGroupInformation
                                            .GetCurrentUser().GetShortUserName());
        }
Esempio n. 3
0
        public virtual void TestAddNewExcludePathToConfiguration()
        {
            Configuration conf = new Configuration();

            rm = new MockRM(conf);
            rm.Start();
            MockNM         nm1     = rm.RegisterNode("host1:1234", 5120);
            MockNM         nm2     = rm.RegisterNode("host2:5678", 10240);
            ClusterMetrics metrics = ClusterMetrics.GetMetrics();

            System.Diagnostics.Debug.Assert((metrics != null));
            int initialMetricCount = metrics.GetNumDecommisionedNMs();
            NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);

            NUnit.Framework.Assert.AreEqual(NodeAction.Normal, nodeHeartbeat.GetNodeAction());
            nodeHeartbeat = nm2.NodeHeartbeat(true);
            NUnit.Framework.Assert.AreEqual(NodeAction.Normal, nodeHeartbeat.GetNodeAction());
            WriteToHostsFile("host2");
            conf.Set(YarnConfiguration.RmNodesExcludeFilePath, hostFile.GetAbsolutePath());
            rm.GetNodesListManager().RefreshNodes(conf);
            CheckDecommissionedNMCount(rm, ++initialMetricCount);
            nodeHeartbeat = nm1.NodeHeartbeat(true);
            NUnit.Framework.Assert.AreEqual("Node should not have been decomissioned.", NodeAction
                                            .Normal, nodeHeartbeat.GetNodeAction());
            nodeHeartbeat = nm2.NodeHeartbeat(true);
            NUnit.Framework.Assert.AreEqual("Node should have been decomissioned but is in state"
                                            + nodeHeartbeat.GetNodeAction(), NodeAction.Shutdown, nodeHeartbeat.GetNodeAction
                                                ());
        }
Esempio n. 4
0
        /// <exception cref="System.Exception"/>
        public virtual void TestTransitionedToStandbyShouldNotHang()
        {
            configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
            Configuration      conf     = new YarnConfiguration(configuration);
            MemoryRMStateStore memStore = new _MemoryRMStateStore_464();

            memStore.Init(conf);
            rm = new _MockRM_472(conf, memStore);
            rm.Init(conf);
            HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo
                                                                       (HAServiceProtocol.RequestSource.RequestByUser);
            NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing
                                            , rm.adminService.GetServiceStatus().GetState());
            NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started"
                                           , rm.adminService.GetServiceStatus().IsReadyToBecomeActive());
            CheckMonitorHealth();
            rm.Start();
            CheckMonitorHealth();
            CheckStandbyRMFunctionality();
            // 2. Transition to Active.
            rm.adminService.TransitionToActive(requestInfo);
            // 3. Try Transition to standby
            Sharpen.Thread t = new Sharpen.Thread(new _Runnable_498(this));
            // TODO Auto-generated catch block
            t.Start();
            rm.GetRMContext().GetStateStore().UpdateApplicationState(null);
            t.Join();
            // wait for thread to finish
            rm.adminService.TransitionToStandby(requestInfo);
            CheckStandbyRMFunctionality();
            rm.Stop();
        }
Esempio n. 5
0
        /// <exception cref="System.Exception"/>
        public virtual void TestInvalidatedAMHostPortOnAMRestart()
        {
            MockRM rm1 = new MockRM(conf);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            // a failed app
            RMApp  app2 = rm1.SubmitApp(200);
            MockAM am2  = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);

            nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
            am2.WaitForState(RMAppAttemptState.Failed);
            rm1.WaitForState(app2.GetApplicationId(), RMAppState.Accepted);
            // before new attempt is launched, the app report returns the invalid AM
            // host and port.
            GetApplicationReportRequest request1 = GetApplicationReportRequest.NewInstance(app2
                                                                                           .GetApplicationId());
            ApplicationReport report1 = rm1.GetClientRMService().GetApplicationReport(request1
                                                                                      ).GetApplicationReport();

            NUnit.Framework.Assert.AreEqual("N/A", report1.GetHost());
            NUnit.Framework.Assert.AreEqual(-1, report1.GetRpcPort());
        }
Esempio n. 6
0
        public virtual void TestNodeRegistrationVersionLessThanRM()
        {
            WriteToHostsFile("host2");
            Configuration conf = new Configuration();

            conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
            conf.Set(YarnConfiguration.RmNodemanagerMinimumVersion, "EqualToRM");
            rm = new MockRM(conf);
            rm.Start();
            string nmVersion = "1.9.9";
            ResourceTrackerService     resourceTrackerService = rm.GetResourceTrackerService();
            RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
                                                                                            >();
            NodeId   nodeId     = NodeId.NewInstance("host2", 1234);
            Resource capability = BuilderUtils.NewResource(1024, 1);

            req.SetResource(capability);
            req.SetNodeId(nodeId);
            req.SetHttpPort(1234);
            req.SetNMVersion(nmVersion);
            // trying to register a invalid node.
            RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager
                                                       (req);

            NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response.GetNodeAction());
            NUnit.Framework.Assert.IsTrue("Diagnostic message did not contain: 'Disallowed NodeManager "
                                          + "Version " + nmVersion + ", is less than the minimum version'", response.GetDiagnosticsMessage
                                              ().Contains("Disallowed NodeManager Version " + nmVersion + ", is less than the minimum version "
                                                          ));
        }
Esempio n. 7
0
        public virtual void TestAdminRefreshQueuesWithFileSystemBasedConfigurationProvider
            ()
        {
            configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
                              );
            //upload default configurations
            UploadDefaultConfiguration();
            try
            {
                rm = new MockRM(configuration);
                rm.Init(configuration);
                rm.Start();
            }
            catch (Exception)
            {
                NUnit.Framework.Assert.Fail("Should not get any exceptions");
            }
            CapacityScheduler cs = (CapacityScheduler)rm.GetRMContext().GetScheduler();
            int maxAppsBefore    = cs.GetConfiguration().GetMaximumSystemApplications();
            CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();

            csConf.Set("yarn.scheduler.capacity.maximum-applications", "5000");
            UploadConfiguration(csConf, "capacity-scheduler.xml");
            rm.adminService.RefreshQueues(RefreshQueuesRequest.NewInstance());
            int maxAppsAfter = cs.GetConfiguration().GetMaximumSystemApplications();

            NUnit.Framework.Assert.AreEqual(maxAppsAfter, 5000);
            NUnit.Framework.Assert.IsTrue(maxAppsAfter != maxAppsBefore);
        }
Esempio n. 8
0
        public virtual void TestHandleContainerStatusInvalidCompletions()
        {
            rm = new MockRM(new YarnConfiguration());
            rm.Start();
            EventHandler handler = Org.Mockito.Mockito.Spy(rm.GetRMContext().GetDispatcher().
                                                           GetEventHandler());
            // Case 1: Unmanaged AM
            RMApp app = rm.SubmitApp(1024, true);
            // Case 1.1: AppAttemptId is null
            NMContainerStatus report = NMContainerStatus.NewInstance(ContainerId.NewContainerId
                                                                         (ApplicationAttemptId.NewInstance(app.GetApplicationId(), 2), 1), ContainerState
                                                                     .Complete, Resource.NewInstance(1024, 1), "Dummy Completed", 0, Priority.NewInstance
                                                                         (10), 1234);

            rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
            Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
                                                                                     )Matchers.Any());
            // Case 1.2: Master container is null
            RMAppAttemptImpl currentAttempt = (RMAppAttemptImpl)app.GetCurrentAppAttempt();

            currentAttempt.SetMasterContainer(null);
            report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(currentAttempt.
                                                                              GetAppAttemptId(), 0), ContainerState.Complete, Resource.NewInstance(1024, 1), "Dummy Completed"
                                                   , 0, Priority.NewInstance(10), 1234);
            rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
            Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
                                                                                     )Matchers.Any());
            // Case 2: Managed AM
            app = rm.SubmitApp(1024);
            // Case 2.1: AppAttemptId is null
            report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(ApplicationAttemptId
                                                                              .NewInstance(app.GetApplicationId(), 2), 1), ContainerState.Complete, Resource.NewInstance
                                                       (1024, 1), "Dummy Completed", 0, Priority.NewInstance(10), 1234);
            try
            {
                rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
            }
            catch (Exception)
            {
            }
            // expected - ignore
            Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
                                                                                     )Matchers.Any());
            // Case 2.2: Master container is null
            currentAttempt = (RMAppAttemptImpl)app.GetCurrentAppAttempt();
            currentAttempt.SetMasterContainer(null);
            report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(currentAttempt.
                                                                              GetAppAttemptId(), 0), ContainerState.Complete, Resource.NewInstance(1024, 1), "Dummy Completed"
                                                   , 0, Priority.NewInstance(10), 1234);
            try
            {
                rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
            }
            catch (Exception)
            {
            }
            // expected - ignore
            Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
                                                                                     )Matchers.Any());
        }
Esempio n. 9
0
        /// <exception cref="System.Exception"/>
        public virtual void TestInvalidContainerReleaseRequest()
        {
            MockRM rm = new MockRM(conf);

            try
            {
                rm.Start();
                // Register node1
                MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
                // Submit an application
                RMApp app1 = rm.SubmitApp(1024);
                // kick the scheduling
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
                MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());
                am1.RegisterAppAttempt();
                am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
                AllocateResponse alloc1Response = am1.Schedule();
                // send the request
                // kick the scheduler
                nm1.NodeHeartbeat(true);
                while (alloc1Response.GetAllocatedContainers().Count < 1)
                {
                    Log.Info("Waiting for containers to be created for app 1...");
                    Sharpen.Thread.Sleep(1000);
                    alloc1Response = am1.Schedule();
                }
                NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0);
                RMApp app2 = rm.SubmitApp(1024);
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
                MockAM       am2      = rm.SendAMLaunched(attempt2.GetAppAttemptId());
                am2.RegisterAppAttempt();
                // Now trying to release container allocated for app1 -> appAttempt1.
                ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId();
                am2.AddContainerToBeReleased(cId);
                try
                {
                    am2.Schedule();
                    NUnit.Framework.Assert.Fail("Exception was expected!!");
                }
                catch (InvalidContainerReleaseException e)
                {
                    StringBuilder sb = new StringBuilder("Cannot release container : ");
                    sb.Append(cId.ToString());
                    sb.Append(" not belonging to this application attempt : ");
                    sb.Append(attempt2.GetAppAttemptId().ToString());
                    NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString()));
                }
            }
            finally
            {
                if (rm != null)
                {
                    rm.Stop();
                }
            }
        }
Esempio n. 10
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAppOnMultiNode()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            conf.Set("yarn.scheduler.capacity.node-locality-delay", "-1");
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
            MockNM nm2 = rm.RegisterNode("h2:5678", 10240);
            RMApp  app = rm.SubmitApp(2000);

            //kick the scheduling
            nm1.NodeHeartbeat(true);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            MockAM       am      = rm.SendAMLaunched(attempt.GetAppAttemptId());

            am.RegisterAppAttempt();
            //request for containers
            int request = 13;

            am.Allocate("h1", 1000, request, new AList <ContainerId>());
            //kick the scheduler
            IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
                                                                                            >()).GetAllocatedContainers();
            int contReceived = conts.Count;

            while (contReceived < 3)
            {
                //only 3 containers are available on node1
                nm1.NodeHeartbeat(true);
                Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
                                                              <ContainerId>()).GetAllocatedContainers());
                contReceived = conts.Count;
                Log.Info("Got " + contReceived + " containers. Waiting to get " + 3);
                Sharpen.Thread.Sleep(WaitSleepMs);
            }
            NUnit.Framework.Assert.AreEqual(3, conts.Count);
            //send node2 heartbeat
            conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
                        ();
            contReceived = conts.Count;
            while (contReceived < 10)
            {
                nm2.NodeHeartbeat(true);
                Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
                                                              <ContainerId>()).GetAllocatedContainers());
                contReceived = conts.Count;
                Log.Info("Got " + contReceived + " containers. Waiting to get " + 10);
                Sharpen.Thread.Sleep(WaitSleepMs);
            }
            NUnit.Framework.Assert.AreEqual(10, conts.Count);
            am.UnregisterAppAttempt();
            nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
            am.WaitForState(RMAppAttemptState.Finished);
            rm.Stop();
        }
Esempio n. 11
0
        // Other stuff is verified in the regular web-services related tests
        /// <summary>Test to verify the following RM HA transitions to the following states.</summary>
        /// <remarks>
        /// Test to verify the following RM HA transitions to the following states.
        /// 1. Standby: Should be a no-op
        /// 2. Active: Active services should start
        /// 3. Active: Should be a no-op.
        /// While active, submit a couple of jobs
        /// 4. Standby: Active services should stop
        /// 5. Active: Active services should start
        /// 6. Stop the RM: All services should stop and RM should not be ready to
        /// become Active
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestFailoverAndTransitions()
        {
            configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
            Configuration conf = new YarnConfiguration(configuration);

            rm = new MockRM(conf);
            rm.Init(conf);
            HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo
                                                                       (HAServiceProtocol.RequestSource.RequestByUser);
            NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing
                                            , rm.adminService.GetServiceStatus().GetState());
            NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started"
                                           , rm.adminService.GetServiceStatus().IsReadyToBecomeActive());
            CheckMonitorHealth();
            rm.Start();
            CheckMonitorHealth();
            CheckStandbyRMFunctionality();
            VerifyClusterMetrics(0, 0, 0, 0, 0, 0);
            // 1. Transition to Standby - must be a no-op
            rm.adminService.TransitionToStandby(requestInfo);
            CheckMonitorHealth();
            CheckStandbyRMFunctionality();
            VerifyClusterMetrics(0, 0, 0, 0, 0, 0);
            // 2. Transition to active
            rm.adminService.TransitionToActive(requestInfo);
            CheckMonitorHealth();
            CheckActiveRMFunctionality();
            VerifyClusterMetrics(1, 1, 1, 1, 2048, 1);
            // 3. Transition to active - no-op
            rm.adminService.TransitionToActive(requestInfo);
            CheckMonitorHealth();
            CheckActiveRMFunctionality();
            VerifyClusterMetrics(1, 2, 2, 2, 2048, 2);
            // 4. Transition to standby
            rm.adminService.TransitionToStandby(requestInfo);
            CheckMonitorHealth();
            CheckStandbyRMFunctionality();
            VerifyClusterMetrics(0, 0, 0, 0, 0, 0);
            // 5. Transition to active to check Active->Standby->Active works
            rm.adminService.TransitionToActive(requestInfo);
            CheckMonitorHealth();
            CheckActiveRMFunctionality();
            VerifyClusterMetrics(1, 1, 1, 1, 2048, 1);
            // 6. Stop the RM. All services should stop and RM should not be ready to
            // become active
            rm.Stop();
            NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Stopping
                                            , rm.adminService.GetServiceStatus().GetState());
            NUnit.Framework.Assert.IsFalse("RM is ready to become active even after it is stopped"
                                           , rm.adminService.GetServiceStatus().IsReadyToBecomeActive());
            NUnit.Framework.Assert.IsFalse("Active RM services are started", rm.AreActiveServicesRunning
                                               ());
            CheckMonitorHealth();
        }
Esempio n. 12
0
        public virtual void TestRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider
            ()
        {
            configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
                              );
            string[]             defaultTestUserGroups = new string[] { "dummy_group1", "dummy_group2" };
            UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("dummyUser",
                                                                                 defaultTestUserGroups);
            string         user          = ugi.GetUserName();
            IList <string> groupWithInit = new AList <string>(2);

            for (int i = 0; i < ugi.GetGroupNames().Length; i++)
            {
                groupWithInit.AddItem(ugi.GetGroupNames()[i]);
            }
            // upload default configurations
            UploadDefaultConfiguration();
            Configuration conf = new Configuration();

            conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestRMAdminService.MockUnixGroupsMapping
                                                                                     ), typeof(GroupMappingServiceProvider));
            UploadConfiguration(conf, "core-site.xml");
            try
            {
                rm = new MockRM(configuration);
                rm.Init(configuration);
                rm.Start();
            }
            catch (Exception)
            {
                NUnit.Framework.Assert.Fail("Should not get any exceptions");
            }
            // Make sure RM will use the updated GroupMappingServiceProvider
            IList <string> groupBefore = new AList <string>(Groups.GetUserToGroupsMappingService
                                                                (configuration).GetGroups(user));

            NUnit.Framework.Assert.IsTrue(groupBefore.Contains("test_group_A") && groupBefore
                                          .Contains("test_group_B") && groupBefore.Contains("test_group_C") && groupBefore
                                          .Count == 3);
            NUnit.Framework.Assert.IsTrue(groupWithInit.Count != groupBefore.Count);
            NUnit.Framework.Assert.IsFalse(groupWithInit.Contains("test_group_A") || groupWithInit
                                           .Contains("test_group_B") || groupWithInit.Contains("test_group_C"));
            // update the groups
            TestRMAdminService.MockUnixGroupsMapping.UpdateGroups();
            rm.adminService.RefreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest.NewInstance
                                                            ());
            IList <string> groupAfter = Groups.GetUserToGroupsMappingService(configuration).GetGroups
                                            (user);

            // should get the updated groups
            NUnit.Framework.Assert.IsTrue(groupAfter.Contains("test_group_D") && groupAfter.Contains
                                              ("test_group_E") && groupAfter.Contains("test_group_F") && groupAfter.Count == 3
                                          );
        }
Esempio n. 13
0
        public virtual void TestAuthorizedAccess()
        {
            TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
                                                                          ();
            rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager);
            rm.Start();
            MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
            IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
                                                                               , string>(2);

            acls[ApplicationAccessType.ViewApp] = "*";
            RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);

            nm1.NodeHeartbeat(true);
            int waitCount = 0;

            while (containerManager.containerTokens == null && waitCount++ < 20)
            {
                Log.Info("Waiting for AM Launch to happen..");
                Sharpen.Thread.Sleep(1000);
            }
            NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
            RMAppAttempt         attempt = app.GetCurrentAppAttempt();
            ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();

            WaitForLaunchedState(attempt);
            // Create a client to the RM.
            Configuration        conf        = rm.GetConfig();
            YarnRPC              rpc         = YarnRPC.Create(conf);
            UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
                                                                                     .ToString());
            Credentials credentials   = containerManager.GetContainerCredentials();
            IPEndPoint  rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();

            Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
                                                                                 .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
            currentUser.AddToken(amRMToken);
            ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this
                                                                                          , rpc, conf));
            RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
                                                       <RegisterApplicationMasterRequest>();
            RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request
                                                                                          );

            NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
            if (UserGroupInformation.IsSecurityEnabled())
            {
                NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
                                                   ()).Length > 0);
            }
            NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs
                                                ()[ApplicationAccessType.ViewApp]);
        }
Esempio n. 14
0
        public virtual void TestDecommissionWithExcludeHosts()
        {
            Configuration conf = new Configuration();

            conf.Set(YarnConfiguration.RmNodesExcludeFilePath, hostFile.GetAbsolutePath());
            WriteToHostsFile(string.Empty);
            DrainDispatcher dispatcher = new DrainDispatcher();

            rm = new _MockRM_162(dispatcher, conf);
            rm.Start();
            MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
            MockNM nm2 = rm.RegisterNode("host2:5678", 10240);
            MockNM nm3 = rm.RegisterNode("localhost:4433", 1024);

            dispatcher.Await();
            int metricCount = ClusterMetrics.GetMetrics().GetNumDecommisionedNMs();
            NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);

            NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
                                                                       ()));
            nodeHeartbeat = nm2.NodeHeartbeat(true);
            NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
                                                                       ()));
            dispatcher.Await();
            // To test that IPs also work
            string ip = NetUtils.NormalizeHostName("localhost");

            WriteToHostsFile("host2", ip);
            rm.GetNodesListManager().RefreshNodes(conf);
            CheckDecommissionedNMCount(rm, metricCount + 2);
            nodeHeartbeat = nm1.NodeHeartbeat(true);
            NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
                                                                       ()));
            nodeHeartbeat = nm2.NodeHeartbeat(true);
            NUnit.Framework.Assert.IsTrue("The decommisioned metrics are not updated", NodeAction
                                          .Shutdown.Equals(nodeHeartbeat.GetNodeAction()));
            nodeHeartbeat = nm3.NodeHeartbeat(true);
            NUnit.Framework.Assert.IsTrue("The decommisioned metrics are not updated", NodeAction
                                          .Shutdown.Equals(nodeHeartbeat.GetNodeAction()));
            dispatcher.Await();
            WriteToHostsFile(string.Empty);
            rm.GetNodesListManager().RefreshNodes(conf);
            nm3 = rm.RegisterNode("localhost:4433", 1024);
            dispatcher.Await();
            nodeHeartbeat = nm3.NodeHeartbeat(true);
            dispatcher.Await();
            NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
                                                                       ()));
            // decommissined node is 1 since 1 node is rejoined after updating exclude
            // file
            CheckDecommissionedNMCount(rm, metricCount + 1);
        }
Esempio n. 15
0
        // This is to test AM Host and rpc port are invalidated after the am attempt
        // is killed or failed, so that client doesn't get the wrong information.
        /// <exception cref="System.Exception"/>
        public virtual void TestInvalidateAMHostPortWhenAMFailedOrKilled()
        {
            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            MockRM rm1 = new MockRM(conf);

            rm1.Start();
            // a succeeded app
            RMApp  app1 = rm1.SubmitApp(200);
            MockNM nm1  = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);

            MockRM.FinishAMAndVerifyAppState(app1, rm1, nm1, am1);
            // a failed app
            RMApp  app2 = rm1.SubmitApp(200);
            MockAM am2  = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);

            nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
            am2.WaitForState(RMAppAttemptState.Failed);
            rm1.WaitForState(app2.GetApplicationId(), RMAppState.Failed);
            // a killed app
            RMApp  app3 = rm1.SubmitApp(200);
            MockAM am3  = MockRM.LaunchAndRegisterAM(app3, rm1, nm1);

            rm1.KillApp(app3.GetApplicationId());
            rm1.WaitForState(app3.GetApplicationId(), RMAppState.Killed);
            rm1.WaitForState(am3.GetApplicationAttemptId(), RMAppAttemptState.Killed);
            GetApplicationsRequest request1 = GetApplicationsRequest.NewInstance(EnumSet.Of(YarnApplicationState
                                                                                            .Finished, YarnApplicationState.Killed, YarnApplicationState.Failed));
            GetApplicationsResponse response1 = rm1.GetClientRMService().GetApplications(request1
                                                                                         );
            IList <ApplicationReport> appList1 = response1.GetApplicationList();

            NUnit.Framework.Assert.AreEqual(3, appList1.Count);
            foreach (ApplicationReport report in appList1)
            {
                // killed/failed apps host and rpc port are invalidated.
                if (report.GetApplicationId().Equals(app2.GetApplicationId()) || report.GetApplicationId
                        ().Equals(app3.GetApplicationId()))
                {
                    NUnit.Framework.Assert.AreEqual("N/A", report.GetHost());
                    NUnit.Framework.Assert.AreEqual(-1, report.GetRpcPort());
                }
                // succeeded app's host and rpc port is not invalidated
                if (report.GetApplicationId().Equals(app1.GetApplicationId()))
                {
                    NUnit.Framework.Assert.IsFalse(report.GetHost().Equals("N/A"));
                    NUnit.Framework.Assert.IsTrue(report.GetRpcPort() != -1);
                }
            }
        }
Esempio n. 16
0
        // Test even if AM container is allocated with containerId not equal to 1, the
        // following allocate requests from AM should be able to retrieve the
        // corresponding NM Token.
        /// <exception cref="System.Exception"/>
        public virtual void TestNMTokenSentForNormalContainer()
        {
            conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName
                         ());
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM       nm1     = rm.RegisterNode("h1:1234", 5120);
            RMApp        app     = rm.SubmitApp(2000);
            RMAppAttempt attempt = app.GetCurrentAppAttempt();
            // Call getNewContainerId to increase container Id so that the AM container
            // Id doesn't equal to one.
            CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();

            cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId();
            // kick the scheduling
            nm1.NodeHeartbeat(true);
            MockAM am = MockRM.LaunchAM(app, rm, nm1);

            // am container Id not equal to 1.
            NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId
                                              () != 1);
            // NMSecretManager doesn't record the node on which the am is allocated.
            NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent
                                               (attempt.GetAppAttemptId(), nm1.GetNodeId()));
            am.RegisterAppAttempt();
            rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
            int NumContainers            = 1;
            IList <Container> containers = new AList <Container>();
            // nmTokens keeps track of all the nmTokens issued in the allocate call.
            IList <NMToken> expectedNMTokens = new AList <NMToken>();

            // am1 allocate 1 container on nm1.
            while (true)
            {
                AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList
                                                        <ContainerId>());
                nm1.NodeHeartbeat(true);
                Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
                Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
                if (containers.Count == NumContainers)
                {
                    break;
                }
                Sharpen.Thread.Sleep(200);
                System.Console.Out.WriteLine("Waiting for container to be allocated.");
            }
            NodeId nodeId = expectedNMTokens[0].GetNodeId();

            // NMToken is sent for the allocated container.
            NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId);
        }
Esempio n. 17
0
        public virtual void TestSetRMIdentifierInRegistration()
        {
            Configuration conf = new Configuration();

            rm = new MockRM(conf);
            rm.Start();
            MockNM nm = new MockNM("host1:1234", 5120, rm.GetResourceTrackerService());
            RegisterNodeManagerResponse response = nm.RegisterNode();

            // Verify the RMIdentifier is correctly set in RegisterNodeManagerResponse
            NUnit.Framework.Assert.AreEqual(ResourceManager.GetClusterTimeStamp(), response.GetRMIdentifier
                                                ());
        }
        /// <exception cref="System.Exception"/>
        public virtual void TestAppCleanupWhenRMRestartedBeforeAppFinished()
        {
            conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
            MemoryRMStateStore memStore = new MemoryRMStateStore();

            memStore.Init(conf);
            // start RM
            MockRM rm1 = new MockRM(conf, memStore);

            rm1.Start();
            MockNM nm1 = new MockNM("127.0.0.1:1234", 1024, rm1.GetResourceTrackerService());

            nm1.RegisterNode();
            MockNM nm2 = new MockNM("127.0.0.1:5678", 1024, rm1.GetResourceTrackerService());

            nm2.RegisterNode();
            // create app and launch the AM
            RMApp  app0 = rm1.SubmitApp(200);
            MockAM am0  = LaunchAM(app0, rm1, nm1);
            // alloc another container on nm2
            AllocateResponse allocResponse = am0.Allocate(Arrays.AsList(ResourceRequest.NewInstance
                                                                            (Priority.NewInstance(1), "*", Resource.NewInstance(1024, 0), 1)), null);

            while (null == allocResponse.GetAllocatedContainers() || allocResponse.GetAllocatedContainers
                       ().IsEmpty())
            {
                nm2.NodeHeartbeat(true);
                allocResponse = am0.Allocate(null, null);
                Sharpen.Thread.Sleep(1000);
            }
            // start new RM
            MockRM rm2 = new MockRM(conf, memStore);

            rm2.Start();
            // nm1/nm2 register to rm2, and do a heartbeat
            nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
            nm1.RegisterNode(Arrays.AsList(NMContainerStatus.NewInstance(ContainerId.NewContainerId
                                                                             (am0.GetApplicationAttemptId(), 1), ContainerState.Complete, Resource.NewInstance
                                                                             (1024, 1), string.Empty, 0, Priority.NewInstance(0), 1234)), Arrays.AsList(app0.
                                                                                                                                                        GetApplicationId()));
            nm2.SetResourceTrackerService(rm2.GetResourceTrackerService());
            nm2.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
            // assert app state has been saved.
            rm2.WaitForState(app0.GetApplicationId(), RMAppState.Failed);
            // wait for application cleanup message received on NM1
            WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
            // wait for application cleanup message received on NM2
            WaitForAppCleanupMessageRecved(nm2, app0.GetApplicationId());
            rm1.Stop();
            rm2.Stop();
        }
Esempio n. 19
0
        public virtual void TestRefreshSuperUserGroupsWithFileSystemBasedConfigurationProvider
            ()
        {
            configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
                              );
            //upload default configurations
            UploadDefaultConfiguration();
            try
            {
                rm = new MockRM(configuration);
                rm.Init(configuration);
                rm.Start();
            }
            catch (Exception)
            {
                NUnit.Framework.Assert.Fail("Should not get any exceptions");
            }
            Configuration coreConf = new Configuration(false);

            coreConf.Set("hadoop.proxyuser.test.groups", "test_groups");
            coreConf.Set("hadoop.proxyuser.test.hosts", "test_hosts");
            UploadConfiguration(coreConf, "core-site.xml");
            rm.adminService.RefreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequest
                                                                .NewInstance());
            NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
                                              ()["hadoop.proxyuser.test.groups"].Count == 1);
            NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
                                              ()["hadoop.proxyuser.test.groups"].Contains("test_groups"));
            NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
                                              ()["hadoop.proxyuser.test.hosts"].Count == 1);
            NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
                                              ()["hadoop.proxyuser.test.hosts"].Contains("test_hosts"));
            Configuration yarnConf = new Configuration(false);

            yarnConf.Set("yarn.resourcemanager.proxyuser.test.groups", "test_groups_1");
            yarnConf.Set("yarn.resourcemanager.proxyuser.test.hosts", "test_hosts_1");
            UploadConfiguration(yarnConf, "yarn-site.xml");
            // RM specific configs will overwrite the common ones
            rm.adminService.RefreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequest
                                                                .NewInstance());
            NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
                                              ()["hadoop.proxyuser.test.groups"].Count == 1);
            NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
                                              ()["hadoop.proxyuser.test.groups"].Contains("test_groups_1"));
            NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
                                              ()["hadoop.proxyuser.test.hosts"].Count == 1);
            NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
                                              ()["hadoop.proxyuser.test.hosts"].Contains("test_hosts_1"));
        }
Esempio n. 20
0
 public virtual void TestRefreshNodesWithLocalConfigurationProvider()
 {
     rm = new MockRM(configuration);
     rm.Init(configuration);
     rm.Start();
     try
     {
         rm.adminService.RefreshNodes(RefreshNodesRequest.NewInstance());
     }
     catch (Exception)
     {
         NUnit.Framework.Assert.Fail("Using localConfigurationProvider. Should not get any exception."
                                     );
     }
 }
Esempio n. 21
0
        public virtual void TestGetNewAppId()
        {
            Logger rootLogger = LogManager.GetRootLogger();

            rootLogger.SetLevel(Level.Debug);
            MockRM rm = new MockRM(conf);

            rm.Start();
            GetNewApplicationResponse resp = rm.GetNewAppId();

            System.Diagnostics.Debug.Assert((resp.GetApplicationId().GetId() != 0));
            System.Diagnostics.Debug.Assert((resp.GetMaximumResourceCapability().GetMemory()
                                             > 0));
            rm.Stop();
        }
Esempio n. 22
0
 /// <exception cref="System.IO.IOException"/>
 protected internal virtual void StartRMs(MockRM rm1, Configuration confForRM1, MockRM
                                          rm2, Configuration confForRM2)
 {
     rm1.Init(confForRM1);
     rm1.Start();
     NUnit.Framework.Assert.IsTrue(rm1.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState
                                   .Standby);
     rm2.Init(confForRM2);
     rm2.Start();
     NUnit.Framework.Assert.IsTrue(rm2.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState
                                   .Standby);
     rm1.adminService.TransitionToActive(requestInfo);
     NUnit.Framework.Assert.IsTrue(rm1.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState
                                   .Active);
 }
Esempio n. 23
0
        public virtual void TestRMDispatcherForHA()
        {
            string errorMessageForEventHandler = "Expect to get the same number of handlers";
            string errorMessageForService      = "Expect to get the same number of services";

            configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
            Configuration conf = new YarnConfiguration(configuration);

            rm = new _MockRM_313(conf);
            rm.Init(conf);
            int expectedEventHandlerCount = ((TestRMHA.MyCountingDispatcher)rm.GetRMContext()
                                             .GetDispatcher()).GetEventHandlerCount();
            int expectedServiceCount = rm.GetServices().Count;

            NUnit.Framework.Assert.IsTrue(expectedEventHandlerCount != 0);
            HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo
                                                                       (HAServiceProtocol.RequestSource.RequestByUser);
            NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing
                                            , rm.adminService.GetServiceStatus().GetState());
            NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started"
                                           , rm.adminService.GetServiceStatus().IsReadyToBecomeActive());
            rm.Start();
            //call transitions to standby and active a couple of times
            rm.adminService.TransitionToStandby(requestInfo);
            rm.adminService.TransitionToActive(requestInfo);
            rm.adminService.TransitionToStandby(requestInfo);
            rm.adminService.TransitionToActive(requestInfo);
            rm.adminService.TransitionToStandby(requestInfo);
            TestRMHA.MyCountingDispatcher dispatcher = (TestRMHA.MyCountingDispatcher)rm.GetRMContext
                                                           ().GetDispatcher();
            NUnit.Framework.Assert.IsTrue(!dispatcher.IsStopped());
            rm.adminService.TransitionToActive(requestInfo);
            NUnit.Framework.Assert.AreEqual(errorMessageForEventHandler, expectedEventHandlerCount
                                            , ((TestRMHA.MyCountingDispatcher)rm.GetRMContext().GetDispatcher()).GetEventHandlerCount
                                                ());
            NUnit.Framework.Assert.AreEqual(errorMessageForService, expectedServiceCount, rm.
                                            GetServices().Count);
            // Keep the dispatcher reference before transitioning to standby
            dispatcher = (TestRMHA.MyCountingDispatcher)rm.GetRMContext().GetDispatcher();
            rm.adminService.TransitionToStandby(requestInfo);
            NUnit.Framework.Assert.AreEqual(errorMessageForEventHandler, expectedEventHandlerCount
                                            , ((TestRMHA.MyCountingDispatcher)rm.GetRMContext().GetDispatcher()).GetEventHandlerCount
                                                ());
            NUnit.Framework.Assert.AreEqual(errorMessageForService, expectedServiceCount, rm.
                                            GetServices().Count);
            NUnit.Framework.Assert.IsTrue(dispatcher.IsStopped());
            rm.Stop();
        }
Esempio n. 24
0
        /// <summary>
        /// Test RM read NM next heartBeat Interval correctly from Configuration file,
        /// and NM get next heartBeat Interval from RM correctly
        /// </summary>
        /// <exception cref="System.Exception"/>
        public virtual void TestGetNextHeartBeatInterval()
        {
            Configuration conf = new Configuration();

            conf.Set(YarnConfiguration.RmNmHeartbeatIntervalMs, "4000");
            rm = new MockRM(conf);
            rm.Start();
            MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
            MockNM nm2 = rm.RegisterNode("host2:5678", 10240);
            NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);

            NUnit.Framework.Assert.AreEqual(4000, nodeHeartbeat.GetNextHeartBeatInterval());
            NodeHeartbeatResponse nodeHeartbeat2 = nm2.NodeHeartbeat(true);

            NUnit.Framework.Assert.AreEqual(4000, nodeHeartbeat2.GetNextHeartBeatInterval());
        }
Esempio n. 25
0
        /// <exception cref="System.Exception"/>
        public virtual void TestMoveSuccessful()
        {
            MockRM rm1 = new MockRM(conf);

            rm1.Start();
            RMApp           app             = rm1.SubmitApp(1024);
            ClientRMService clientRMService = rm1.GetClientRMService();

            // FIFO scheduler does not support moves
            clientRMService.MoveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest.NewInstance
                                                            (app.GetApplicationId(), "newqueue"));
            RMApp rmApp = rm1.GetRMContext().GetRMApps()[app.GetApplicationId()];

            NUnit.Framework.Assert.AreEqual("newqueue", rmApp.GetQueue());
            rm1.Stop();
        }
        /// <exception cref="System.Exception"/>
        public virtual void TestUsageWithOneAttemptAndOneContainer()
        {
            MockRM rm = new MockRM(conf);

            rm.Start();
            MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());

            nm.RegisterNode();
            RMApp        app0         = rm.SubmitApp(200);
            RMAppMetrics rmAppMetrics = app0.GetRMAppMetrics();

            NUnit.Framework.Assert.IsTrue("Before app submittion, memory seconds should have been 0 but was "
                                          + rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() == 0);
            NUnit.Framework.Assert.IsTrue("Before app submission, vcore seconds should have been 0 but was "
                                          + rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() == 0);
            RMAppAttempt attempt0 = app0.GetCurrentAppAttempt();

            nm.NodeHeartbeat(true);
            MockAM am0 = rm.SendAMLaunched(attempt0.GetAppAttemptId());

            am0.RegisterAppAttempt();
            RMContainer rmContainer = rm.GetResourceScheduler().GetRMContainer(attempt0.GetMasterContainer
                                                                                   ().GetId());
            // Allow metrics to accumulate.
            int sleepInterval       = 1000;
            int cumulativeSleepTime = 0;

            while (rmAppMetrics.GetMemorySeconds() <= 0 && cumulativeSleepTime < 5000)
            {
                Sharpen.Thread.Sleep(sleepInterval);
                cumulativeSleepTime += sleepInterval;
            }
            rmAppMetrics = app0.GetRMAppMetrics();
            NUnit.Framework.Assert.IsTrue("While app is running, memory seconds should be >0 but is "
                                          + rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() > 0);
            NUnit.Framework.Assert.IsTrue("While app is running, vcore seconds should be >0 but is "
                                          + rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() > 0);
            MockRM.FinishAMAndVerifyAppState(app0, rm, nm, am0);
            AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(rmContainer);

            rmAppMetrics = app0.GetRMAppMetrics();
            NUnit.Framework.Assert.AreEqual("Unexcpected MemorySeconds value", ru.GetMemorySeconds
                                                (), rmAppMetrics.GetMemorySeconds());
            NUnit.Framework.Assert.AreEqual("Unexpected VcoreSeconds value", ru.GetVcoreSeconds
                                                (), rmAppMetrics.GetVcoreSeconds());
            rm.Stop();
        }
Esempio n. 27
0
        /// <exception cref="System.Exception"/>
        public virtual void TestResourceTypes()
        {
            Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes> >
            driver = new Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
                                                                 > >();
            CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();

            csconf.SetResourceComparator(typeof(DominantResourceCalculator));
            YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf);

            testCapacityDRConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
                                                                              ), typeof(ResourceScheduler));
            YarnConfiguration testCapacityDefConf = new YarnConfiguration();

            testCapacityDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
                                                                               ), typeof(ResourceScheduler));
            YarnConfiguration testFairDefConf = new YarnConfiguration();

            testFairDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(FairScheduler), typeof(
                                         ResourceScheduler));
            driver[conf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory);
            driver[testCapacityDRConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.
                                                    Cpu, YarnServiceProtos.SchedulerResourceTypes.Memory);
            driver[testCapacityDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes
                                                     .Memory);
            driver[testFairDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory
                                                 , YarnServiceProtos.SchedulerResourceTypes.Cpu);
            foreach (KeyValuePair <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
                                                               > > entry in driver)
            {
                EnumSet <YarnServiceProtos.SchedulerResourceTypes> expectedValue = entry.Value;
                MockRM rm = new MockRM(entry.Key);
                rm.Start();
                MockNM nm1  = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
                RMApp  app1 = rm.SubmitApp(2048);
                nm1.NodeHeartbeat(true);
                RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
                MockAM       am1      = rm.SendAMLaunched(attempt1.GetAppAttemptId());
                RegisterApplicationMasterResponse resp = am1.RegisterAppAttempt();
                EnumSet <YarnServiceProtos.SchedulerResourceTypes> types = resp.GetSchedulerResourceTypes
                                                                               ();
                Log.Info("types = " + types.ToString());
                NUnit.Framework.Assert.AreEqual(expectedValue, types);
                rm.Stop();
            }
        }
Esempio n. 28
0
        public virtual void TestFailoverClearsRMContext()
        {
            configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
            configuration.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
            Configuration      conf     = new YarnConfiguration(configuration);
            MemoryRMStateStore memStore = new MemoryRMStateStore();

            memStore.Init(conf);
            // 1. start RM
            rm = new MockRM(conf, memStore);
            rm.Init(conf);
            rm.Start();
            HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo
                                                                       (HAServiceProtocol.RequestSource.RequestByUser);
            CheckMonitorHealth();
            CheckStandbyRMFunctionality();
            // 2. Transition to active
            rm.adminService.TransitionToActive(requestInfo);
            CheckMonitorHealth();
            CheckActiveRMFunctionality();
            VerifyClusterMetrics(1, 1, 1, 1, 2048, 1);
            NUnit.Framework.Assert.AreEqual(1, rm.GetRMContext().GetRMNodes().Count);
            NUnit.Framework.Assert.AreEqual(1, rm.GetRMContext().GetRMApps().Count);
            // 3. Create new RM
            rm = new _MockRM_550(conf, memStore);
            rm.Init(conf);
            rm.Start();
            CheckMonitorHealth();
            CheckStandbyRMFunctionality();
            // 4. Try Transition to active, throw exception
            try
            {
                rm.adminService.TransitionToActive(requestInfo);
                NUnit.Framework.Assert.Fail("Transitioned to Active should throw exception.");
            }
            catch (Exception e)
            {
                NUnit.Framework.Assert.IsTrue("Error when transitioning to Active mode".Contains(
                                                  e.Message));
            }
            // 5. Clears the metrics
            VerifyClusterMetrics(0, 0, 0, 0, 0, 0);
            NUnit.Framework.Assert.AreEqual(0, rm.GetRMContext().GetRMNodes().Count);
            NUnit.Framework.Assert.AreEqual(0, rm.GetRMContext().GetRMApps().Count);
        }
Esempio n. 29
0
        public virtual void TestDecommissionWithIncludeHosts()
        {
            WriteToHostsFile("localhost", "host1", "host2");
            Configuration conf = new Configuration();

            conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
            rm = new MockRM(conf);
            rm.Start();
            MockNM         nm1     = rm.RegisterNode("host1:1234", 5120);
            MockNM         nm2     = rm.RegisterNode("host2:5678", 10240);
            MockNM         nm3     = rm.RegisterNode("localhost:4433", 1024);
            ClusterMetrics metrics = ClusterMetrics.GetMetrics();

            System.Diagnostics.Debug.Assert((metrics != null));
            int metricCount = metrics.GetNumDecommisionedNMs();
            NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);

            NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
                                                                       ()));
            nodeHeartbeat = nm2.NodeHeartbeat(true);
            NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
                                                                       ()));
            nodeHeartbeat = nm3.NodeHeartbeat(true);
            NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
                                                                       ()));
            // To test that IPs also work
            string ip = NetUtils.NormalizeHostName("localhost");

            WriteToHostsFile("host1", ip);
            rm.GetNodesListManager().RefreshNodes(conf);
            CheckDecommissionedNMCount(rm, ++metricCount);
            nodeHeartbeat = nm1.NodeHeartbeat(true);
            NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
                                                                       ()));
            NUnit.Framework.Assert.AreEqual(1, ClusterMetrics.GetMetrics().GetNumDecommisionedNMs
                                                ());
            nodeHeartbeat = nm2.NodeHeartbeat(true);
            NUnit.Framework.Assert.IsTrue("Node is not decommisioned.", NodeAction.Shutdown.Equals
                                              (nodeHeartbeat.GetNodeAction()));
            nodeHeartbeat = nm3.NodeHeartbeat(true);
            NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
                                                                       ()));
            NUnit.Framework.Assert.AreEqual(metricCount, ClusterMetrics.GetMetrics().GetNumDecommisionedNMs
                                                ());
        }
Esempio n. 30
0
        /// <exception cref="System.Exception"/>
        public virtual void TestFinishApplicationMasterBeforeRegistering()
        {
            MockRM rm = new MockRM(conf);

            try
            {
                rm.Start();
                // Register node1
                MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
                // Submit an application
                RMApp  app1 = rm.SubmitApp(2048);
                MockAM am1  = MockRM.LaunchAM(app1, rm, nm1);
                FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
                                                                                                .Failed, string.Empty, string.Empty);
                try
                {
                    am1.UnregisterAppAttempt(req, false);
                    NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
                                                );
                }
                catch (ApplicationMasterNotRegisteredException e)
                {
                    NUnit.Framework.Assert.IsNotNull(e);
                    NUnit.Framework.Assert.IsNotNull(e.Message);
                    NUnit.Framework.Assert.IsTrue(e.Message.Contains("Application Master is trying to unregister before registering for:"
                                                                     ));
                }
                catch (Exception)
                {
                    NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
                                                );
                }
                am1.RegisterAppAttempt();
                am1.UnregisterAppAttempt(req, false);
                am1.WaitForState(RMAppAttemptState.Finishing);
            }
            finally
            {
                if (rm != null)
                {
                    rm.Stop();
                }
            }
        }