Ejemplo n.º 1
0
 /// <exception cref="System.Exception"/>
 public virtual void TestRemoveLabelsFromNode()
 {
     mgr.AddToCluserNodeLabels(ToSet("p1", "p2", "p3"));
     mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1"), ToNodeId("n2"
                                                                                   ), ToSet("p2"), ToNodeId("n3"), ToSet("p3")));
     // active one NM to n1:1
     mgr.ActivateNode(NodeId.NewInstance("n1", 1), SmallResource);
     try
     {
         mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1")));
         NUnit.Framework.Assert.Fail("removeLabelsFromNode should trigger IOException");
     }
     catch (IOException)
     {
     }
     mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1")));
     try
     {
         mgr.RemoveLabelsFromNode(ImmutableMap.Of(ToNodeId("n1:1"), ToSet("p1")));
     }
     catch (IOException e)
     {
         NUnit.Framework.Assert.Fail("IOException from removeLabelsFromNode " + e);
     }
 }
Ejemplo n.º 2
0
 /// <exception cref="System.Exception"/>
 public virtual void TestLabelsToNodesOnNodeActiveDeactive()
 {
     // Activate a node without assigning any labels
     mgr.ActivateNode(NodeId.NewInstance("n1", 1), Org.Apache.Hadoop.Yarn.Api.Records.Resource
                      .NewInstance(10, 0));
     NUnit.Framework.Assert.IsTrue(mgr.GetLabelsToNodes().IsEmpty());
     AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels
                                                                                 ()));
     // Add labels and replace labels on node
     mgr.AddToCluserNodeLabels(ToSet("p1"));
     mgr.ReplaceLabelsOnNode(ImmutableMap.Of(ToNodeId("n1"), ToSet("p1")));
     // p1 -> n1, n1:1
     NUnit.Framework.Assert.AreEqual(2, mgr.GetLabelsToNodes()["p1"].Count);
     AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels
                                                                                 ()));
     // Activate a node for which host to label mapping exists
     mgr.ActivateNode(NodeId.NewInstance("n1", 2), Org.Apache.Hadoop.Yarn.Api.Records.Resource
                      .NewInstance(10, 0));
     // p1 -> n1, n1:1, n1:2
     NUnit.Framework.Assert.AreEqual(3, mgr.GetLabelsToNodes()["p1"].Count);
     AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels
                                                                                 ()));
     // Deactivate a node. n1:1 will be removed from the map
     mgr.DeactivateNode(NodeId.NewInstance("n1", 1));
     // p1 -> n1, n1:2
     NUnit.Framework.Assert.AreEqual(2, mgr.GetLabelsToNodes()["p1"].Count);
     AssertLabelsToNodesEquals(mgr.GetLabelsToNodes(), TransposeNodeToLabels(mgr.GetNodeLabels
                                                                                 ()));
 }
Ejemplo n.º 3
0
        public AMAttemptInfo(AMInfo amInfo, string jobId, string user)
        {
            this.nodeHttpAddress = string.Empty;
            this.nodeId          = string.Empty;
            string nmHost     = amInfo.GetNodeManagerHost();
            int    nmHttpPort = amInfo.GetNodeManagerHttpPort();
            int    nmPort     = amInfo.GetNodeManagerPort();

            if (nmHost != null)
            {
                this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
                NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
                this.nodeId = nodeId.ToString();
            }
            this.id          = amInfo.GetAppAttemptId().GetAttemptId();
            this.startTime   = amInfo.GetStartTime();
            this.containerId = string.Empty;
            this.logsLink    = string.Empty;
            ContainerId containerId = amInfo.GetContainerId();

            if (containerId != null)
            {
                this.containerId = containerId.ToString();
                this.logsLink    = StringHelper.Join(MRWebAppUtil.GetYARNWebappScheme() + nodeHttpAddress
                                                     , StringHelper.Ujoin("node", "containerlogs", this.containerId, user));
            }
        }
Ejemplo n.º 4
0
 /// <summary>Get mapping of nodes to labels</summary>
 /// <returns>nodes to labels map</returns>
 public virtual IDictionary <NodeId, ICollection <string> > GetNodeLabels()
 {
     try
     {
         readLock.Lock();
         IDictionary <NodeId, ICollection <string> > nodeToLabels = new Dictionary <NodeId, ICollection
                                                                                    <string> >();
         foreach (KeyValuePair <string, CommonNodeLabelsManager.Host> entry in nodeCollections)
         {
             string hostName = entry.Key;
             CommonNodeLabelsManager.Host host = entry.Value;
             foreach (NodeId nodeId in host.nms.Keys)
             {
                 ICollection <string> nodeLabels = GetLabelsByNode(nodeId);
                 if (nodeLabels == null || nodeLabels.IsEmpty())
                 {
                     continue;
                 }
                 nodeToLabels[nodeId] = nodeLabels;
             }
             if (!host.labels.IsEmpty())
             {
                 nodeToLabels[NodeId.NewInstance(hostName, WildcardPort)] = host.labels;
             }
         }
         return(Sharpen.Collections.UnmodifiableMap(nodeToLabels));
     }
     finally
     {
         readLock.Unlock();
     }
 }
Ejemplo n.º 5
0
 public virtual void Handle(ContainerAllocatorEvent @event)
 {
     if (@event.GetType() == ContainerAllocator.EventType.ContainerReq)
     {
         Log.Info("Processing the event " + @event.ToString());
         // Assign the same container ID as the AM
         ContainerId cID = ContainerId.NewContainerId(GetContext().GetApplicationAttemptId
                                                          (), this.containerId.GetContainerId());
         Container container = recordFactory.NewRecordInstance <Container>();
         container.SetId(cID);
         NodeId nodeId = NodeId.NewInstance(this.nmHost, this.nmPort);
         container.SetNodeId(nodeId);
         container.SetContainerToken(null);
         container.SetNodeHttpAddress(this.nmHost + ":" + this.nmHttpPort);
         // send the container-assigned event to task attempt
         if (@event.GetAttemptID().GetTaskId().GetTaskType() == TaskType.Map)
         {
             JobCounterUpdateEvent jce = new JobCounterUpdateEvent(@event.GetAttemptID().GetTaskId
                                                                       ().GetJobId());
             // TODO Setting OTHER_LOCAL_MAP for now.
             jce.AddCounterUpdate(JobCounter.OtherLocalMaps, 1);
             eventHandler.Handle(jce);
         }
         eventHandler.Handle(new TaskAttemptContainerAssignedEvent(@event.GetAttemptID(),
                                                                   container, applicationACLs));
     }
 }
Ejemplo n.º 6
0
 /// <exception cref="System.Exception"/>
 public virtual void TestActivateNodeManagerWithZeroPort()
 {
     // active two NM, one is zero port , another is non-zero port. no exception
     // should be raised
     mgr.ActivateNode(NodeId.NewInstance("n1", 0), SmallResource);
     mgr.ActivateNode(NodeId.NewInstance("n1", 2), LargeNode);
 }
Ejemplo n.º 7
0
        public virtual void TestLogsViewSingle()
        {
            Log.Info("HsLogsPage with params for single log and data limits");
            MockAppContext ctx = new MockAppContext(0, 1, 1, 1);
            IDictionary <string, string> @params = new Dictionary <string, string>();
            Configuration conf = new YarnConfiguration();

            conf.SetBoolean(YarnConfiguration.LogAggregationEnabled, true);
            @params["start"] = "-2048";
            @params["end"]   = "-1024";
            @params[YarnWebParams.ContainerLogType] = "syslog";
            @params[YarnWebParams.ContainerId]      = MRApp.NewContainerId(1, 1, 333, 1).ToString(
                );
            @params[YarnWebParams.NmNodename] = NodeId.NewInstance(MockJobs.NmHost, MockJobs.
                                                                   NmPort).ToString();
            @params[YarnWebParams.EntityString] = "container_10_0001_01_000001";
            @params[YarnWebParams.AppOwner]     = "owner";
            Injector injector = WebAppTests.TestPage <AppContext>(typeof(AggregatedLogsPage),
                                                                  ctx, @params, new _AbstractModule_201(conf));
            PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);

            Org.Mockito.Mockito.Verify(spyPw).Write("Logs not available for container_10_0001_01_000001."
                                                    + " Aggregation may not be complete, " + "Check back later or try the nodemanager at "
                                                    + MockJobs.NmHost + ":" + MockJobs.NmPort);
        }
Ejemplo n.º 8
0
        public virtual void TestNMTokenIdentifier()
        {
            ApplicationAttemptId appAttemptId = ApplicationAttemptId.NewInstance(ApplicationId
                                                                                 .NewInstance(1, 1), 1);
            NodeId            nodeId = NodeId.NewInstance("host0", 0);
            string            applicationSubmitter = "usr0";
            int               masterKeyId          = 1;
            NMTokenIdentifier token = new NMTokenIdentifier(appAttemptId, nodeId, applicationSubmitter
                                                            , masterKeyId);
            NMTokenIdentifier anotherToken = new NMTokenIdentifier();

            byte[]          tokenContent = token.GetBytes();
            DataInputBuffer dib          = new DataInputBuffer();

            dib.Reset(tokenContent, tokenContent.Length);
            anotherToken.ReadFields(dib);
            // verify the whole record equals with original record
            NUnit.Framework.Assert.AreEqual("Token is not the same after serialization " + "and deserialization."
                                            , token, anotherToken);
            // verify all properties are the same as original
            NUnit.Framework.Assert.AreEqual("appAttemptId from proto is not the same with original token"
                                            , anotherToken.GetApplicationAttemptId(), appAttemptId);
            NUnit.Framework.Assert.AreEqual("NodeId from proto is not the same with original token"
                                            , anotherToken.GetNodeId(), nodeId);
            NUnit.Framework.Assert.AreEqual("applicationSubmitter from proto is not the same with original token"
                                            , anotherToken.GetApplicationSubmitter(), applicationSubmitter);
            NUnit.Framework.Assert.AreEqual("masterKeyId from proto is not the same with original token"
                                            , anotherToken.GetKeyId(), masterKeyId);
        }
Ejemplo n.º 9
0
        private static RMContainer CreateRMContainer(ContainerId containerId)
        {
            RMContainer container = Org.Mockito.Mockito.Mock <RMContainer>();

            Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(containerId);
            Org.Mockito.Mockito.When(container.GetAllocatedNode()).ThenReturn(NodeId.NewInstance
                                                                                  ("test host", -100));
            Org.Mockito.Mockito.When(container.GetAllocatedResource()).ThenReturn(Resource.NewInstance
                                                                                      (-1, -1));
            Org.Mockito.Mockito.When(container.GetAllocatedPriority()).ThenReturn(Priority.Undefined
                                                                                  );
            Org.Mockito.Mockito.When(container.GetCreationTime()).ThenReturn(int.MaxValue + 1L
                                                                             );
            Org.Mockito.Mockito.When(container.GetFinishTime()).ThenReturn(int.MaxValue + 2L);
            Org.Mockito.Mockito.When(container.GetDiagnosticsInfo()).ThenReturn("test diagnostics info"
                                                                                );
            Org.Mockito.Mockito.When(container.GetContainerExitStatus()).ThenReturn(-1);
            Org.Mockito.Mockito.When(container.GetContainerState()).ThenReturn(ContainerState
                                                                               .Complete);
            Container mockContainer = Org.Mockito.Mockito.Mock <Container>();

            Org.Mockito.Mockito.When(container.GetContainer()).ThenReturn(mockContainer);
            Org.Mockito.Mockito.When(mockContainer.GetNodeHttpAddress()).ThenReturn("http://localhost:1234"
                                                                                    );
            return(container);
        }
Ejemplo n.º 10
0
        public virtual void TestReconnect()
        {
            string   hostname1  = "localhost1";
            Resource capability = BuilderUtils.NewResource(1024, 1);
            RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                   >();
            NodeId nodeId1 = NodeId.NewInstance(hostname1, 0);

            request1.SetNodeId(nodeId1);
            request1.SetHttpPort(0);
            request1.SetResource(capability);
            resourceTrackerService.RegisterNodeManager(request1);
            NUnit.Framework.Assert.AreEqual(RMNodeEventType.Started, rmNodeEvents[0].GetType(
                                                ));
            rmNodeEvents.Clear();
            resourceTrackerService.RegisterNodeManager(request1);
            NUnit.Framework.Assert.AreEqual(RMNodeEventType.Reconnected, rmNodeEvents[0].GetType
                                                ());
            rmNodeEvents.Clear();
            resourceTrackerService.RegisterNodeManager(request1);
            capability = BuilderUtils.NewResource(1024, 2);
            request1.SetResource(capability);
            NUnit.Framework.Assert.AreEqual(RMNodeEventType.Reconnected, rmNodeEvents[0].GetType
                                                ());
        }
Ejemplo n.º 11
0
        public AMAttemptInfo(AMInfo amInfo, string jobId, string user, string host, string
                             pathPrefix)
        {
            this.nodeHttpAddress = string.Empty;
            this.nodeId          = string.Empty;
            string nmHost     = amInfo.GetNodeManagerHost();
            int    nmHttpPort = amInfo.GetNodeManagerHttpPort();
            int    nmPort     = amInfo.GetNodeManagerPort();

            if (nmHost != null)
            {
                this.nodeHttpAddress = nmHost + ":" + nmHttpPort;
                NodeId nodeId = NodeId.NewInstance(nmHost, nmPort);
                this.nodeId = nodeId.ToString();
            }
            this.id            = amInfo.GetAppAttemptId().GetAttemptId();
            this.startTime     = amInfo.GetStartTime();
            this.containerId   = string.Empty;
            this.logsLink      = string.Empty;
            this.shortLogsLink = string.Empty;
            ContainerId containerId = amInfo.GetContainerId();

            if (containerId != null)
            {
                this.containerId = containerId.ToString();
                this.logsLink    = StringHelper.Join(host, pathPrefix, StringHelper.Ujoin("logs", this
                                                                                          .nodeId, this.containerId, jobId, user));
                this.shortLogsLink = StringHelper.Ujoin("logs", this.nodeId, this.containerId, jobId
                                                        , user);
            }
        }
Ejemplo n.º 12
0
        public virtual void TestNodeRegistrationVersionLessThanRM()
        {
            WriteToHostsFile("host2");
            Configuration conf = new Configuration();

            conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
            conf.Set(YarnConfiguration.RmNodemanagerMinimumVersion, "EqualToRM");
            rm = new MockRM(conf);
            rm.Start();
            string nmVersion = "1.9.9";
            ResourceTrackerService     resourceTrackerService = rm.GetResourceTrackerService();
            RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
                                                                                            >();
            NodeId   nodeId     = NodeId.NewInstance("host2", 1234);
            Resource capability = BuilderUtils.NewResource(1024, 1);

            req.SetResource(capability);
            req.SetNodeId(nodeId);
            req.SetHttpPort(1234);
            req.SetNMVersion(nmVersion);
            // trying to register a invalid node.
            RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager
                                                       (req);

            NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response.GetNodeAction());
            NUnit.Framework.Assert.IsTrue("Diagnostic message did not contain: 'Disallowed NodeManager "
                                          + "Version " + nmVersion + ", is less than the minimum version'", response.GetDiagnosticsMessage
                                              ().Contains("Disallowed NodeManager Version " + nmVersion + ", is less than the minimum version "
                                                          ));
        }
Ejemplo n.º 13
0
        public virtual void TestUpdateMaxAllocationUsesTotal()
        {
            int configuredMaxVCores = 20;
            int configuredMaxMemory = 10 * 1024;

            Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                                    .NewInstance(configuredMaxMemory, configuredMaxVCores);
            ConfigureScheduler();
            YarnConfiguration conf = GetConf();

            conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
                        );
            conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
                        );
            conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
            MockRM rm = new MockRM(conf);

            try
            {
                rm.Start();
                AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
                    );
                Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                            .NewInstance(0, 0);
                Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                            .NewInstance(1024, 5);
                Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
                                                                            .NewInstance(2048, 10);
                SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>();
                Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo"
                                                                                              , 8080));
                Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource
                                                                                      );
                Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1);
                SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>();
                Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar"
                                                                                              , 8081));
                Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource
                                                                                      );
                Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2);
                VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
                scheduler.nodes = new Dictionary <NodeId, SchedulerNode>();
                scheduler.nodes[mockNode1.GetNodeID()] = mockNode1;
                scheduler.UpdateMaximumAllocation(mockNode1, true);
                VerifyMaximumResourceCapability(fullResource1, scheduler);
                scheduler.nodes[mockNode2.GetNodeID()] = mockNode2;
                scheduler.UpdateMaximumAllocation(mockNode2, true);
                VerifyMaximumResourceCapability(fullResource2, scheduler);
                Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID());
                scheduler.UpdateMaximumAllocation(mockNode2, false);
                VerifyMaximumResourceCapability(fullResource1, scheduler);
                Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID());
                scheduler.UpdateMaximumAllocation(mockNode1, false);
                VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
            }
            finally
            {
                rm.Stop();
            }
        }
Ejemplo n.º 14
0
        public virtual void TestRegisterNodeManagerRequest()
        {
            ApplicationId        appId           = ApplicationId.NewInstance(123456789, 1);
            ApplicationAttemptId attemptId       = ApplicationAttemptId.NewInstance(appId, 1);
            ContainerId          containerId     = ContainerId.NewContainerId(attemptId, 1);
            NMContainerStatus    containerReport = NMContainerStatus.NewInstance(containerId, ContainerState
                                                                                 .Running, Resource.NewInstance(1024, 1), "diagnostics", 0, Priority.NewInstance(
                                                                                     10), 1234);
            IList <NMContainerStatus>  reports = Arrays.AsList(containerReport);
            RegisterNodeManagerRequest request = RegisterNodeManagerRequest.NewInstance(NodeId
                                                                                        .NewInstance("1.1.1.1", 1000), 8080, Resource.NewInstance(1024, 1), "NM-version-id"
                                                                                        , reports, Arrays.AsList(appId));
            RegisterNodeManagerRequest requestProto = new RegisterNodeManagerRequestPBImpl(((
                                                                                                RegisterNodeManagerRequestPBImpl)request).GetProto());

            NUnit.Framework.Assert.AreEqual(containerReport, requestProto.GetNMContainerStatuses
                                                ()[0]);
            NUnit.Framework.Assert.AreEqual(8080, requestProto.GetHttpPort());
            NUnit.Framework.Assert.AreEqual("NM-version-id", requestProto.GetNMVersion());
            NUnit.Framework.Assert.AreEqual(NodeId.NewInstance("1.1.1.1", 1000), requestProto
                                            .GetNodeId());
            NUnit.Framework.Assert.AreEqual(Resource.NewInstance(1024, 1), requestProto.GetResource
                                                ());
            NUnit.Framework.Assert.AreEqual(1, requestProto.GetRunningApplications().Count);
            NUnit.Framework.Assert.AreEqual(appId, requestProto.GetRunningApplications()[0]);
        }
Ejemplo n.º 15
0
        /// <exception cref="System.Exception"/>
        public virtual void TestContainerAllocateWithLabels()
        {
            // set node -> label
            mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
            mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
                                                .NewInstance("h2", 0), ToSet("y")));
            // inject node label manager
            MockRM rm1 = new _MockRM_638(this, GetConfigurationWithQueueLabels(conf));

            rm1.GetRMContext().SetNodeLabelManager(mgr);
            rm1.Start();
            MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
            // label = x
            MockNM nm2 = rm1.RegisterNode("h2:1234", 8000);
            // label = y
            MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
            // label = <empty>
            ContainerId containerId;
            // launch an app to queue a1 (label = x), and check all container will
            // be allocated in h1
            RMApp  app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
            MockAM am1  = MockRM.LaunchAndRegisterAM(app1, rm1, nm3);

            // request a container.
            am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
            containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2);
            NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
                                                            .Allocated, 10 * 1000));
            NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
                                                           .Allocated, 10 * 1000));
            CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h1");
            // launch an app to queue b1 (label = y), and check all container will
            // be allocated in h2
            RMApp  app2 = rm1.SubmitApp(200, "app", "user", null, "b1");
            MockAM am2  = MockRM.LaunchAndRegisterAM(app2, rm1, nm3);

            // request a container.
            am2.Allocate("*", 1024, 1, new AList <ContainerId>(), "y");
            containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2);
            NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState
                                                            .Allocated, 10 * 1000));
            NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState
                                                           .Allocated, 10 * 1000));
            CheckTaskContainersHost(am2.GetApplicationAttemptId(), containerId, rm1, "h2");
            // launch an app to queue c1 (label = ""), and check all container will
            // be allocated in h3
            RMApp  app3 = rm1.SubmitApp(200, "app", "user", null, "c1");
            MockAM am3  = MockRM.LaunchAndRegisterAM(app3, rm1, nm3);

            // request a container.
            am3.Allocate("*", 1024, 1, new AList <ContainerId>());
            containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2);
            NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
                                                            .Allocated, 10 * 1000));
            NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
                                                           .Allocated, 10 * 1000));
            CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3");
            rm1.Close();
        }
Ejemplo n.º 16
0
        private static Container CreateMockContainer()
        {
            Container container = Org.Mockito.Mockito.Mock <Container>();
            NodeId    nodeId    = NodeId.NewInstance("foo.bar.org", 1234);

            Org.Mockito.Mockito.When(container.GetNodeId()).ThenReturn(nodeId);
            return(container);
        }
        private Token CreateNewContainerToken(ContainerId contId, string containerManagerAddr
                                              )
        {
            long currentTime = Runtime.CurrentTimeMillis();

            return(MRApp.NewContainerToken(NodeId.NewInstance("127.0.0.1", 1234), Sharpen.Runtime.GetBytesForString
                                               ("password"), new ContainerTokenIdentifier(contId, containerManagerAddr, "user",
                                                                                          Resource.NewInstance(1024, 1), currentTime + 10000L, 123, currentTime, Priority.
                                                                                          NewInstance(0), 0)));
        }
Ejemplo n.º 18
0
            public virtual IList <NodeReport> CreateFakeNodeReports()
            {
                NodeId     nodeId = NodeId.NewInstance("localhost", 0);
                NodeReport report = NodeReport.NewInstance(nodeId, NodeState.Running, "localhost"
                                                           , "rack1", null, null, 4, null, 1000l, null);
                IList <NodeReport> reports = new AList <NodeReport>();

                reports.AddItem(report);
                return(reports);
            }
Ejemplo n.º 19
0
        private IDictionary <string, CommonNodeLabelsManager.Host> CloneNodeMap()
        {
            ICollection <NodeId> nodesToCopy = new HashSet <NodeId>();

            foreach (string nodeName in nodeCollections.Keys)
            {
                nodesToCopy.AddItem(NodeId.NewInstance(nodeName, WildcardPort));
            }
            return(CloneNodeMap(nodesToCopy));
        }
Ejemplo n.º 20
0
        /// <exception cref="System.Exception"/>
        private void TestRPCTimeout(string rpcClass)
        {
            Configuration conf = new Configuration();

            // set timeout low for the test
            conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
            conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
            YarnRPC    rpc      = YarnRPC.Create(conf);
            string     bindAddr = "localhost:0";
            IPEndPoint addr     = NetUtils.CreateSocketAddr(bindAddr);
            Server     server   = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager
                                                    (this), addr, conf, null, 1);

            server.Start();
            try
            {
                ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
                                                                                                  ContainerManagementProtocol), server.GetListenerAddress(), conf);
                ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                                 >();
                ApplicationId        applicationId        = ApplicationId.NewInstance(0, 0);
                ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
                                                                                             , 0);
                ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
                NodeId      nodeId      = NodeId.NewInstance("localhost", 1234);
                Resource    resource    = Resource.NewInstance(1234, 2);
                ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
                    containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
                    42, 42, Priority.NewInstance(0), 0);
                Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
                                                                     ("password"), containerTokenIdentifier);
                StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
                                                                                    , containerToken);
                IList <StartContainerRequest> list = new AList <StartContainerRequest>();
                list.AddItem(scRequest);
                StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
                try
                {
                    proxy.StartContainers(allRequests);
                }
                catch (Exception e)
                {
                    Log.Info(StringUtils.StringifyException(e));
                    NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException
                                                                                         ).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName);
                    return;
                }
            }
            finally
            {
                server.Stop();
            }
            NUnit.Framework.Assert.Fail("timeout exception should have occurred!");
        }
Ejemplo n.º 21
0
        private Container MockContainer(int i)
        {
            ApplicationId        appId       = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1);
            ApplicationAttemptId attemptId   = ApplicationAttemptId.NewInstance(appId, 1);
            ContainerId          containerId = ContainerId.NewContainerId(attemptId, i);

            nodeId = NodeId.NewInstance("localhost", 0);
            // Create an empty record
            containerToken = recordFactory.NewRecordInstance <Token>();
            return(Container.NewInstance(containerId, nodeId, null, null, null, containerToken
                                         ));
        }
Ejemplo n.º 22
0
        public virtual void TestCompareRMNodeAfterReconnect()
        {
            Configuration     yarnConf  = new YarnConfiguration();
            CapacityScheduler scheduler = new CapacityScheduler();

            scheduler.SetConf(yarnConf);
            ConfigurationProvider configurationProvider = ConfigurationProviderFactory.GetConfigurationProvider
                                                              (yarnConf);

            configurationProvider.Init(yarnConf);
            context.SetConfigurationProvider(configurationProvider);
            RMNodeLabelsManager nlm = new RMNodeLabelsManager();

            nlm.Init(yarnConf);
            nlm.Start();
            context.SetNodeLabelManager(nlm);
            scheduler.SetRMContext(context);
            scheduler.Init(yarnConf);
            scheduler.Start();
            dispatcher.Register(typeof(SchedulerEventType), scheduler);
            string   hostname1  = "localhost1";
            Resource capability = BuilderUtils.NewResource(4096, 4);
            RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                   >();
            NodeId nodeId1 = NodeId.NewInstance(hostname1, 0);

            request1.SetNodeId(nodeId1);
            request1.SetHttpPort(0);
            request1.SetResource(capability);
            resourceTrackerService.RegisterNodeManager(request1);
            NUnit.Framework.Assert.IsNotNull(context.GetRMNodes()[nodeId1]);
            // verify Scheduler and RMContext use same RMNode reference.
            NUnit.Framework.Assert.IsTrue(scheduler.GetSchedulerNode(nodeId1).GetRMNode() ==
                                          context.GetRMNodes()[nodeId1]);
            NUnit.Framework.Assert.AreEqual(context.GetRMNodes()[nodeId1].GetTotalCapability(
                                                ), capability);
            Resource capability1 = BuilderUtils.NewResource(2048, 2);

            request1.SetResource(capability1);
            resourceTrackerService.RegisterNodeManager(request1);
            NUnit.Framework.Assert.IsNotNull(context.GetRMNodes()[nodeId1]);
            // verify Scheduler and RMContext use same RMNode reference
            // after reconnect.
            NUnit.Framework.Assert.IsTrue(scheduler.GetSchedulerNode(nodeId1).GetRMNode() ==
                                          context.GetRMNodes()[nodeId1]);
            // verify RMNode's capability is changed.
            NUnit.Framework.Assert.AreEqual(context.GetRMNodes()[nodeId1].GetTotalCapability(
                                                ), capability1);
            nlm.Stop();
            scheduler.Stop();
        }
Ejemplo n.º 23
0
					 GetCMProxy(string containerMgrBindAddr, ContainerId containerId)
				{
					IPEndPoint addr = NetUtils.GetConnectAddress(this._enclosing._enclosing.server);
					string containerManagerBindAddr = addr.GetHostName() + ":" + addr.Port;
					Token token = this._enclosing.tokenSecretManager.CreateNMToken(containerId.GetApplicationAttemptId
						(), NodeId.NewInstance(addr.GetHostName(), addr.Port), "user");
					ContainerManagementProtocolProxy cmProxy = new ContainerManagementProtocolProxy(this
						._enclosing._enclosing.conf);
					ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData proxy = new 
						ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData(this, YarnRPC
						.Create(this._enclosing._enclosing.conf), containerManagerBindAddr, containerId, 
						token);
					return proxy;
				}
Ejemplo n.º 24
0
 public virtual NodeId ToNodeId(string str)
 {
     if (str.Contains(":"))
     {
         int    idx = str.IndexOf(':');
         NodeId id  = NodeId.NewInstance(Sharpen.Runtime.Substring(str, 0, idx), Sharpen.Extensions.ValueOf
                                             (Sharpen.Runtime.Substring(str, idx + 1)));
         return(id);
     }
     else
     {
         return(NodeId.NewInstance(str, CommonNodeLabelsManager.WildcardPort));
     }
 }
Ejemplo n.º 25
0
            protected override void ConfigureServlets()
            {
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.conf.Set(YarnConfiguration
                                                                                                .NmLocalDirs, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                                                                .testRootDir.GetAbsolutePath());
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.conf.Set(YarnConfiguration
                                                                                                .NmLogDirs, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                                                                .testLogDir.GetAbsolutePath());
                NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();

                healthChecker.Init(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                   .conf);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.dirsHandler
                    = healthChecker.GetDiskHandler();
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.aclsManager
                    = new ApplicationACLsManager(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                 .conf);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.nmContext
                    = new NodeManager.NMContext(null, null, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                .dirsHandler, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                .aclsManager, null);
                NodeId nodeId = NodeId.NewInstance("testhost.foo.com", 9999);

                ((NodeManager.NMContext)Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                 .nmContext).SetNodeId(nodeId);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.resourceView
                    = new _ResourceView_106();
                // 15.5G in bytes
                // 16G in bytes
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.nmWebApp =
                    new WebServer.NMWebApp(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                           .resourceView, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                           .aclsManager, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                           .dirsHandler);
                this.Bind <JAXBContextResolver>();
                this.Bind <NMWebServices>();
                this.Bind <GenericExceptionHandler>();
                this.Bind <Context>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                 .nmContext);
                this.Bind <WebApp>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                .nmWebApp);
                this.Bind <ResourceView>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                      .resourceView);
                this.Bind <ApplicationACLsManager>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                                .aclsManager);
                this.Bind <LocalDirsHandlerService>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                                 .dirsHandler);
                this.Serve("/*").With(typeof(GuiceContainer));
            }
        public virtual void TestWriteContainer()
        {
            RMContainer container = CreateRMContainer(ContainerId.NewContainerId(ApplicationAttemptId
                                                                                 .NewInstance(ApplicationId.NewInstance(0, 1), 1), 1));

            writer.ContainerStarted(container);
            ContainerHistoryData containerHD = null;

            for (int i = 0; i < MaxRetries; ++i)
            {
                containerHD = store.GetContainer(ContainerId.NewContainerId(ApplicationAttemptId.
                                                                            NewInstance(ApplicationId.NewInstance(0, 1), 1), 1));
                if (containerHD != null)
                {
                    break;
                }
                else
                {
                    Sharpen.Thread.Sleep(100);
                }
            }
            NUnit.Framework.Assert.IsNotNull(containerHD);
            NUnit.Framework.Assert.AreEqual(NodeId.NewInstance("test host", -100), containerHD
                                            .GetAssignedNode());
            NUnit.Framework.Assert.AreEqual(Resource.NewInstance(-1, -1), containerHD.GetAllocatedResource
                                                ());
            NUnit.Framework.Assert.AreEqual(Priority.Undefined, containerHD.GetPriority());
            NUnit.Framework.Assert.AreEqual(0L, container.GetCreationTime());
            writer.ContainerFinished(container);
            for (int i_1 = 0; i_1 < MaxRetries; ++i_1)
            {
                containerHD = store.GetContainer(ContainerId.NewContainerId(ApplicationAttemptId.
                                                                            NewInstance(ApplicationId.NewInstance(0, 1), 1), 1));
                if (containerHD.GetContainerState() != null)
                {
                    break;
                }
                else
                {
                    Sharpen.Thread.Sleep(100);
                }
            }
            NUnit.Framework.Assert.AreEqual("test diagnostics info", containerHD.GetDiagnosticsInfo
                                                ());
            NUnit.Framework.Assert.AreEqual(-1, containerHD.GetContainerExitStatus());
            NUnit.Framework.Assert.AreEqual(ContainerState.Complete, containerHD.GetContainerState
                                                ());
        }
Ejemplo n.º 27
0
        /// <exception cref="System.Exception"/>
        public virtual void TestContainerAllocationWithSingleUserLimits()
        {
            RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();

            mgr.Init(conf);
            // set node -> label
            mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
            mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
                                                .NewInstance("h2", 0), ToSet("y")));
            // inject node label manager
            MockRM rm1 = new _MockRM_471(mgr, GetConfigurationWithDefaultQueueLabels(conf));

            rm1.GetRMContext().SetNodeLabelManager(mgr);
            rm1.Start();
            MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);

            // label = x
            rm1.RegisterNode("h2:1234", 8000);
            // label = y
            MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
            // label = <empty>
            // launch an app to queue a1 (label = x), and check all container will
            // be allocated in h1
            RMApp  app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
            MockAM am1  = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
            // A has only 10% of x, so it can only allocate one container in label=empty
            ContainerId containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
                                                                     ), 2);

            am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty);
            NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
                                                           .Allocated, 10 * 1000));
            // Cannot allocate 2nd label=empty container
            containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 3);
            am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty);
            NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm3, containerId, RMContainerState
                                                            .Allocated, 10 * 1000));
            // A has default user limit = 100, so it can use all resource in label = x
            // We can allocate floor(8000 / 1024) = 7 containers
            for (int id = 3; id <= 8; id++)
            {
                containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), id);
                am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
                NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
                                                               .Allocated, 10 * 1000));
            }
            rm1.Close();
        }
Ejemplo n.º 28
0
        public virtual void TestGetContainerReport()
        {
            ContainerId containerId = ContainerId.NewContainerId(ApplicationAttemptId.NewInstance
                                                                     (ApplicationId.NewInstance(0, 1), 1), 1);
            ContainerReport container;

            if (callerUGI == null)
            {
                container = historyManager.GetContainer(containerId);
            }
            else
            {
                try
                {
                    container = callerUGI.DoAs(new _PrivilegedExceptionAction_302(this, containerId));
                    if (callerUGI != null && callerUGI.GetShortUserName().Equals("user3"))
                    {
                        // The exception is expected
                        NUnit.Framework.Assert.Fail();
                    }
                }
                catch (AuthorizationException e)
                {
                    if (callerUGI != null && callerUGI.GetShortUserName().Equals("user3"))
                    {
                        // The exception is expected
                        return;
                    }
                    throw;
                }
            }
            NUnit.Framework.Assert.IsNotNull(container);
            NUnit.Framework.Assert.AreEqual(int.MaxValue + 1L, container.GetCreationTime());
            NUnit.Framework.Assert.AreEqual(int.MaxValue + 2L, container.GetFinishTime());
            NUnit.Framework.Assert.AreEqual(Resource.NewInstance(-1, -1), container.GetAllocatedResource
                                                ());
            NUnit.Framework.Assert.AreEqual(NodeId.NewInstance("test host", 100), container.GetAssignedNode
                                                ());
            NUnit.Framework.Assert.AreEqual(Priority.Undefined, container.GetPriority());
            NUnit.Framework.Assert.AreEqual("test diagnostics info", container.GetDiagnosticsInfo
                                                ());
            NUnit.Framework.Assert.AreEqual(ContainerState.Complete, container.GetContainerState
                                                ());
            NUnit.Framework.Assert.AreEqual(-1, container.GetContainerExitStatus());
            NUnit.Framework.Assert.AreEqual("http://0.0.0.0:8188/applicationhistory/logs/" +
                                            "test host:100/container_0_0001_01_000001/" + "container_0_0001_01_000001/user1"
                                            , container.GetLogUrl());
        }
Ejemplo n.º 29
0
        /// <exception cref="System.IO.IOException"/>
        public virtual LogParams GetLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID
                                                )
        {
            JobId jobId = TypeConverter.ToYarn(oldJobID);
            GetJobReportRequest request = recordFactory.NewRecordInstance <GetJobReportRequest
                                                                           >();

            request.SetJobId(jobId);
            JobReport report = ((GetJobReportResponse)Invoke("getJobReport", typeof(GetJobReportRequest
                                                                                    ), request)).GetJobReport();

            if (EnumSet.Of(JobState.Succeeded, JobState.Failed, JobState.Killed, JobState.Error
                           ).Contains(report.GetJobState()))
            {
                if (oldTaskAttemptID != null)
                {
                    GetTaskAttemptReportRequest taRequest = recordFactory.NewRecordInstance <GetTaskAttemptReportRequest
                                                                                             >();
                    taRequest.SetTaskAttemptId(TypeConverter.ToYarn(oldTaskAttemptID));
                    TaskAttemptReport taReport = ((GetTaskAttemptReportResponse)Invoke("getTaskAttemptReport"
                                                                                       , typeof(GetTaskAttemptReportRequest), taRequest)).GetTaskAttemptReport();
                    if (taReport.GetContainerId() == null || taReport.GetNodeManagerHost() == null)
                    {
                        throw new IOException("Unable to get log information for task: " + oldTaskAttemptID
                                              );
                    }
                    return(new LogParams(taReport.GetContainerId().ToString(), taReport.GetContainerId
                                             ().GetApplicationAttemptId().GetApplicationId().ToString(), NodeId.NewInstance(taReport
                                                                                                                            .GetNodeManagerHost(), taReport.GetNodeManagerPort()).ToString(), report.GetUser
                                             ()));
                }
                else
                {
                    if (report.GetAMInfos() == null || report.GetAMInfos().Count == 0)
                    {
                        throw new IOException("Unable to get log information for job: " + oldJobID);
                    }
                    AMInfo amInfo = report.GetAMInfos()[report.GetAMInfos().Count - 1];
                    return(new LogParams(amInfo.GetContainerId().ToString(), amInfo.GetAppAttemptId()
                                         .GetApplicationId().ToString(), NodeId.NewInstance(amInfo.GetNodeManagerHost(),
                                                                                            amInfo.GetNodeManagerPort()).ToString(), report.GetUser()));
                }
            }
            else
            {
                throw new IOException("Cannot get log path for a in-progress job");
            }
        }
Ejemplo n.º 30
0
        public virtual void TestRPCResponseId()
        {
            string   node       = "localhost";
            Resource capability = BuilderUtils.NewResource(1024, 1);
            RegisterNodeManagerRequest request = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                  >();

            nodeId = NodeId.NewInstance(node, 1234);
            request.SetNodeId(nodeId);
            request.SetHttpPort(0);
            request.SetResource(capability);
            RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
                                                                                   >();

            request1.SetNodeId(nodeId);
            request1.SetHttpPort(0);
            request1.SetResource(capability);
            resourceTrackerService.RegisterNodeManager(request1);
            NodeStatus nodeStatus = recordFactory.NewRecordInstance <NodeStatus>();

            nodeStatus.SetNodeId(nodeId);
            NodeHealthStatus nodeHealthStatus = recordFactory.NewRecordInstance <NodeHealthStatus
                                                                                 >();

            nodeHealthStatus.SetIsNodeHealthy(true);
            nodeStatus.SetNodeHealthStatus(nodeHealthStatus);
            NodeHeartbeatRequest nodeHeartBeatRequest = recordFactory.NewRecordInstance <NodeHeartbeatRequest
                                                                                         >();

            nodeHeartBeatRequest.SetNodeStatus(nodeStatus);
            nodeStatus.SetResponseId(0);
            NodeHeartbeatResponse response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest
                                                                                  );

            NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 1);
            nodeStatus.SetResponseId(response.GetResponseId());
            response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
            NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 2);
            /* try calling with less response id */
            response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
            NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 2);
            nodeStatus.SetResponseId(0);
            response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
            NUnit.Framework.Assert.IsTrue(NodeAction.Resync.Equals(response.GetNodeAction()));
            NUnit.Framework.Assert.AreEqual("Too far behind rm response id:2 nm response id:0"
                                            , response.GetDiagnosticsMessage());
        }