Esempio n. 1
0
        public virtual void TestReconnect()
        {
            string   hostname1  = "localhost1";
            Resource capability = BuilderUtils.NewResource(1024, 1);
            RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                   >();
            NodeId nodeId1 = NodeId.NewInstance(hostname1, 0);

            request1.SetNodeId(nodeId1);
            request1.SetHttpPort(0);
            request1.SetResource(capability);
            resourceTrackerService.RegisterNodeManager(request1);
            NUnit.Framework.Assert.AreEqual(RMNodeEventType.Started, rmNodeEvents[0].GetType(
                                                ));
            rmNodeEvents.Clear();
            resourceTrackerService.RegisterNodeManager(request1);
            NUnit.Framework.Assert.AreEqual(RMNodeEventType.Reconnected, rmNodeEvents[0].GetType
                                                ());
            rmNodeEvents.Clear();
            resourceTrackerService.RegisterNodeManager(request1);
            capability = BuilderUtils.NewResource(1024, 2);
            request1.SetResource(capability);
            NUnit.Framework.Assert.AreEqual(RMNodeEventType.Reconnected, rmNodeEvents[0].GetType
                                                ());
        }
Esempio n. 2
0
        public virtual void TestCompareRMNodeAfterReconnect()
        {
            Configuration     yarnConf  = new YarnConfiguration();
            CapacityScheduler scheduler = new CapacityScheduler();

            scheduler.SetConf(yarnConf);
            ConfigurationProvider configurationProvider = ConfigurationProviderFactory.GetConfigurationProvider
                                                              (yarnConf);

            configurationProvider.Init(yarnConf);
            context.SetConfigurationProvider(configurationProvider);
            RMNodeLabelsManager nlm = new RMNodeLabelsManager();

            nlm.Init(yarnConf);
            nlm.Start();
            context.SetNodeLabelManager(nlm);
            scheduler.SetRMContext(context);
            scheduler.Init(yarnConf);
            scheduler.Start();
            dispatcher.Register(typeof(SchedulerEventType), scheduler);
            string   hostname1  = "localhost1";
            Resource capability = BuilderUtils.NewResource(4096, 4);
            RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                   >();
            NodeId nodeId1 = NodeId.NewInstance(hostname1, 0);

            request1.SetNodeId(nodeId1);
            request1.SetHttpPort(0);
            request1.SetResource(capability);
            resourceTrackerService.RegisterNodeManager(request1);
            NUnit.Framework.Assert.IsNotNull(context.GetRMNodes()[nodeId1]);
            // verify Scheduler and RMContext use same RMNode reference.
            NUnit.Framework.Assert.IsTrue(scheduler.GetSchedulerNode(nodeId1).GetRMNode() ==
                                          context.GetRMNodes()[nodeId1]);
            NUnit.Framework.Assert.AreEqual(context.GetRMNodes()[nodeId1].GetTotalCapability(
                                                ), capability);
            Resource capability1 = BuilderUtils.NewResource(2048, 2);

            request1.SetResource(capability1);
            resourceTrackerService.RegisterNodeManager(request1);
            NUnit.Framework.Assert.IsNotNull(context.GetRMNodes()[nodeId1]);
            // verify Scheduler and RMContext use same RMNode reference
            // after reconnect.
            NUnit.Framework.Assert.IsTrue(scheduler.GetSchedulerNode(nodeId1).GetRMNode() ==
                                          context.GetRMNodes()[nodeId1]);
            // verify RMNode's capability is changed.
            NUnit.Framework.Assert.AreEqual(context.GetRMNodes()[nodeId1].GetTotalCapability(
                                                ), capability1);
            nlm.Stop();
            scheduler.Stop();
        }
Esempio n. 3
0
        public virtual void TestNodeRegistrationVersionLessThanRM()
        {
            WriteToHostsFile("host2");
            Configuration conf = new Configuration();

            conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
            conf.Set(YarnConfiguration.RmNodemanagerMinimumVersion, "EqualToRM");
            rm = new MockRM(conf);
            rm.Start();
            string nmVersion = "1.9.9";
            ResourceTrackerService     resourceTrackerService = rm.GetResourceTrackerService();
            RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
                                                                                            >();
            NodeId   nodeId     = NodeId.NewInstance("host2", 1234);
            Resource capability = BuilderUtils.NewResource(1024, 1);

            req.SetResource(capability);
            req.SetNodeId(nodeId);
            req.SetHttpPort(1234);
            req.SetNMVersion(nmVersion);
            // trying to register a invalid node.
            RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager
                                                       (req);

            NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response.GetNodeAction());
            NUnit.Framework.Assert.IsTrue("Diagnostic message did not contain: 'Disallowed NodeManager "
                                          + "Version " + nmVersion + ", is less than the minimum version'", response.GetDiagnosticsMessage
                                              ().Contains("Disallowed NodeManager Version " + nmVersion + ", is less than the minimum version "
                                                          ));
        }
Esempio n. 4
0
        public virtual void TestNodeRegistrationWithMinimumAllocations()
        {
            Configuration conf = new Configuration();

            conf.Set(YarnConfiguration.RmSchedulerMinimumAllocationMb, "2048");
            conf.Set(YarnConfiguration.RmSchedulerMinimumAllocationVcores, "4");
            rm = new MockRM(conf);
            rm.Start();
            ResourceTrackerService     resourceTrackerService = rm.GetResourceTrackerService();
            RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
                                                                                            >();
            NodeId nodeId = BuilderUtils.NewNodeId("host", 1234);

            req.SetNodeId(nodeId);
            Resource capability = BuilderUtils.NewResource(1024, 1);

            req.SetResource(capability);
            RegisterNodeManagerResponse response1 = resourceTrackerService.RegisterNodeManager
                                                        (req);

            NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response1.GetNodeAction());
            capability.SetMemory(2048);
            capability.SetVirtualCores(1);
            req.SetResource(capability);
            RegisterNodeManagerResponse response2 = resourceTrackerService.RegisterNodeManager
                                                        (req);

            NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response2.GetNodeAction());
            capability.SetMemory(1024);
            capability.SetVirtualCores(4);
            req.SetResource(capability);
            RegisterNodeManagerResponse response3 = resourceTrackerService.RegisterNodeManager
                                                        (req);

            NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response3.GetNodeAction());
            capability.SetMemory(2048);
            capability.SetVirtualCores(4);
            req.SetResource(capability);
            RegisterNodeManagerResponse response4 = resourceTrackerService.RegisterNodeManager
                                                        (req);

            NUnit.Framework.Assert.AreEqual(NodeAction.Normal, response4.GetNodeAction());
        }
        public virtual void TestRPCResponseId()
        {
            string   node       = "localhost";
            Resource capability = BuilderUtils.NewResource(1024, 1);
            RegisterNodeManagerRequest request = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                  >();

            nodeId = NodeId.NewInstance(node, 1234);
            request.SetNodeId(nodeId);
            request.SetHttpPort(0);
            request.SetResource(capability);
            RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                   >();

            request1.SetNodeId(nodeId);
            request1.SetHttpPort(0);
            request1.SetResource(capability);
            resourceTrackerService.RegisterNodeManager(request1);
            NodeStatus nodeStatus = recordFactory.NewRecordInstance <NodeStatus>();

            nodeStatus.SetNodeId(nodeId);
            NodeHealthStatus nodeHealthStatus = recordFactory.NewRecordInstance <NodeHealthStatus
                                                                                 >();

            nodeHealthStatus.SetIsNodeHealthy(true);
            nodeStatus.SetNodeHealthStatus(nodeHealthStatus);
            NodeHeartbeatRequest nodeHeartBeatRequest = recordFactory.NewRecordInstance <NodeHeartbeatRequest
                                                                                         >();

            nodeHeartBeatRequest.SetNodeStatus(nodeStatus);
            nodeStatus.SetResponseId(0);
            NodeHeartbeatResponse response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest
                                                                                  );

            NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 1);
            nodeStatus.SetResponseId(response.GetResponseId());
            response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
            NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 2);
            /* try calling with less response id */
            response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
            NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 2);
            nodeStatus.SetResponseId(0);
            response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
            NUnit.Framework.Assert.IsTrue(NodeAction.Resync.Equals(response.GetNodeAction()));
            NUnit.Framework.Assert.AreEqual("Too far behind rm response id:2 nm response id:0"
                                            , response.GetDiagnosticsMessage());
        }
Esempio n. 6
0
        public virtual void TestNMExpiry()
        {
            string   hostname1  = "localhost1";
            string   hostname2  = "localhost2";
            string   hostname3  = "localhost3";
            Resource capability = BuilderUtils.NewResource(1024, 1);
            RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                   >();
            NodeId nodeId1 = NodeId.NewInstance(hostname1, 0);

            request1.SetNodeId(nodeId1);
            request1.SetHttpPort(0);
            request1.SetResource(capability);
            resourceTrackerService.RegisterNodeManager(request1);
            RegisterNodeManagerRequest request2 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                   >();
            NodeId nodeId2 = NodeId.NewInstance(hostname2, 0);

            request2.SetNodeId(nodeId2);
            request2.SetHttpPort(0);
            request2.SetResource(capability);
            resourceTrackerService.RegisterNodeManager(request2);
            int waitCount = 0;

            while (ClusterMetrics.GetMetrics().GetNumLostNMs() != 2 && waitCount++ < 20)
            {
                lock (this)
                {
                    Sharpen.Runtime.Wait(this, 100);
                }
            }
            NUnit.Framework.Assert.AreEqual(2, ClusterMetrics.GetMetrics().GetNumLostNMs());
            request3 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest>();
            NodeId nodeId3 = NodeId.NewInstance(hostname3, 0);

            request3.SetNodeId(nodeId3);
            request3.SetHttpPort(0);
            request3.SetResource(capability);
            resourceTrackerService.RegisterNodeManager(request3);
            /* test to see if hostanme 3 does not expire */
            stopT = false;
            new TestNMExpiry.ThirdNodeHeartBeatThread(this).Start();
            NUnit.Framework.Assert.AreEqual(2, ClusterMetrics.GetMetrics().GetNumLostNMs());
            stopT = true;
        }
Esempio n. 7
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public NodeManager(string hostName, int containerManagerPort, int httpPort, string
                           rackName, Resource capability, ResourceManager resourceManager)
        {
            this.containerManagerAddress = hostName + ":" + containerManagerPort;
            this.nodeHttpAddress         = hostName + ":" + httpPort;
            this.rackName = rackName;
            this.resourceTrackerService = resourceManager.GetResourceTrackerService();
            this.capability             = capability;
            Resources.AddTo(available, capability);
            this.nodeId = NodeId.NewInstance(hostName, containerManagerPort);
            RegisterNodeManagerRequest request = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                  >();

            request.SetHttpPort(httpPort);
            request.SetResource(capability);
            request.SetNodeId(this.nodeId);
            request.SetNMVersion(YarnVersionInfo.GetVersion());
            resourceTrackerService.RegisterNodeManager(request);
            this.resourceManager = resourceManager;
            resourceManager.GetResourceScheduler().GetNodeReport(this.nodeId);
        }
Esempio n. 8
0
        /// <exception cref="System.Exception"/>
        public virtual RegisterNodeManagerResponse RegisterNode(IList <NMContainerStatus>
                                                                containerReports, IList <ApplicationId> runningApplications)
        {
            RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
                                                                                            >();

            req.SetNodeId(nodeId);
            req.SetHttpPort(httpPort);
            Resource resource = BuilderUtils.NewResource(memory, vCores);

            req.SetResource(resource);
            req.SetContainerStatuses(containerReports);
            req.SetNMVersion(version);
            req.SetRunningApplications(runningApplications);
            RegisterNodeManagerResponse registrationResponse = resourceTracker.RegisterNodeManager
                                                                   (req);

            this.currentContainerTokenMasterKey = registrationResponse.GetContainerTokenMasterKey
                                                      ();
            this.currentNMTokenMasterKey = registrationResponse.GetNMTokenMasterKey();
            return(registrationResponse);
        }
Esempio n. 9
0
        public virtual void TestNodeRegistrationSuccess()
        {
            WriteToHostsFile("host2");
            Configuration conf = new Configuration();

            conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
            rm = new MockRM(conf);
            rm.Start();
            ResourceTrackerService     resourceTrackerService = rm.GetResourceTrackerService();
            RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
                                                                                            >();
            NodeId   nodeId     = NodeId.NewInstance("host2", 1234);
            Resource capability = BuilderUtils.NewResource(1024, 1);

            req.SetResource(capability);
            req.SetNodeId(nodeId);
            req.SetHttpPort(1234);
            req.SetNMVersion(YarnVersionInfo.GetVersion());
            // trying to register a invalid node.
            RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager
                                                       (req);

            NUnit.Framework.Assert.AreEqual(NodeAction.Normal, response.GetNodeAction());
        }