public static TaskTrackerInfo FromYarn(NodeReport node) { TaskTrackerInfo taskTracker = new TaskTrackerInfo(TtNamePrefix + node.GetNodeId() .ToString()); return(taskTracker); }
/// <summary>Prints the node report for node id.</summary> /// <param name="nodeIdStr"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> /// <exception cref="System.IO.IOException"/> private void PrintNodeStatus(string nodeIdStr) { NodeId nodeId = ConverterUtils.ToNodeId(nodeIdStr); IList <NodeReport> nodesReport = client.GetNodeReports(); // Use PrintWriter.println, which uses correct platform line ending. ByteArrayOutputStream baos = new ByteArrayOutputStream(); PrintWriter nodeReportStr = new PrintWriter(new OutputStreamWriter(baos, Sharpen.Extensions.GetEncoding ("UTF-8"))); NodeReport nodeReport = null; foreach (NodeReport report in nodesReport) { if (!report.GetNodeId().Equals(nodeId)) { continue; } nodeReport = report; nodeReportStr.WriteLine("Node Report : "); nodeReportStr.Write("\tNode-Id : "); nodeReportStr.WriteLine(nodeReport.GetNodeId()); nodeReportStr.Write("\tRack : "); nodeReportStr.WriteLine(nodeReport.GetRackName()); nodeReportStr.Write("\tNode-State : "); nodeReportStr.WriteLine(nodeReport.GetNodeState()); nodeReportStr.Write("\tNode-Http-Address : "); nodeReportStr.WriteLine(nodeReport.GetHttpAddress()); nodeReportStr.Write("\tLast-Health-Update : "); nodeReportStr.WriteLine(DateFormatUtils.Format(Sharpen.Extensions.CreateDate(nodeReport .GetLastHealthReportTime()), "E dd/MMM/yy hh:mm:ss:SSzz")); nodeReportStr.Write("\tHealth-Report : "); nodeReportStr.WriteLine(nodeReport.GetHealthReport()); nodeReportStr.Write("\tContainers : "); nodeReportStr.WriteLine(nodeReport.GetNumContainers()); nodeReportStr.Write("\tMemory-Used : "); nodeReportStr.WriteLine((nodeReport.GetUsed() == null) ? "0MB" : (nodeReport.GetUsed ().GetMemory() + "MB")); nodeReportStr.Write("\tMemory-Capacity : "); nodeReportStr.WriteLine(nodeReport.GetCapability().GetMemory() + "MB"); nodeReportStr.Write("\tCPU-Used : "); nodeReportStr.WriteLine((nodeReport.GetUsed() == null) ? "0 vcores" : (nodeReport .GetUsed().GetVirtualCores() + " vcores")); nodeReportStr.Write("\tCPU-Capacity : "); nodeReportStr.WriteLine(nodeReport.GetCapability().GetVirtualCores() + " vcores"); nodeReportStr.Write("\tNode-Labels : "); // Create a List for node labels since we need it get sorted IList <string> nodeLabelsList = new AList <string>(report.GetNodeLabels()); nodeLabelsList.Sort(); nodeReportStr.WriteLine(StringUtils.Join(nodeLabelsList.GetEnumerator(), ',')); } if (nodeReport == null) { nodeReportStr.Write("Could not find the node report for node id : " + nodeIdStr); } nodeReportStr.Close(); sysout.WriteLine(baos.ToString("UTF-8")); }
public virtual void TestAMRMUnusableNodes() { MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 10000); MockNM nm2 = rm.RegisterNode("127.0.0.2:1234", 10000); MockNM nm3 = rm.RegisterNode("127.0.0.3:1234", 10000); MockNM nm4 = rm.RegisterNode("127.0.0.4:1234", 10000); dispatcher.Await(); RMApp app1 = rm.SubmitApp(2000); // Trigger the scheduling so the AM gets 'launched' on nm1 nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); // register AM returns no unusable node am1.RegisterAppAttempt(); // allocate request returns no updated node AllocateRequest allocateRequest1 = AllocateRequest.NewInstance(0, 0F, null, null, null); AllocateResponse response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1 ); IList <NodeReport> updatedNodes = response1.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count); SyncNodeHeartbeat(nm4, false); // allocate request returns updated node allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null , null, null); response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1); updatedNodes = response1.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count); NodeReport nr = updatedNodes.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId()); NUnit.Framework.Assert.AreEqual(NodeState.Unhealthy, nr.GetNodeState()); // resending the allocate request returns the same result response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1); updatedNodes = response1.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count); nr = updatedNodes.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId()); NUnit.Framework.Assert.AreEqual(NodeState.Unhealthy, nr.GetNodeState()); SyncNodeLost(nm3); // subsequent allocate request returns delta allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null , null, null); response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1); updatedNodes = response1.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count); nr = updatedNodes.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(nm3.GetNodeId(), nr.GetNodeId()); NUnit.Framework.Assert.AreEqual(NodeState.Lost, nr.GetNodeState()); // registering another AM gives it the complete failed list RMApp app2 = rm.SubmitApp(2000); // Trigger nm2 heartbeat so that AM gets launched on it nm2.NodeHeartbeat(true); RMAppAttempt attempt2 = app2.GetCurrentAppAttempt(); MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId()); // register AM returns all unusable nodes am2.RegisterAppAttempt(); // allocate request returns no updated node AllocateRequest allocateRequest2 = AllocateRequest.NewInstance(0, 0F, null, null, null); AllocateResponse response2 = Allocate(attempt2.GetAppAttemptId(), allocateRequest2 ); updatedNodes = response2.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count); SyncNodeHeartbeat(nm4, true); // both AM's should get delta updated nodes allocateRequest1 = AllocateRequest.NewInstance(response1.GetResponseId(), 0F, null , null, null); response1 = Allocate(attempt1.GetAppAttemptId(), allocateRequest1); updatedNodes = response1.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count); nr = updatedNodes.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId()); NUnit.Framework.Assert.AreEqual(NodeState.Running, nr.GetNodeState()); allocateRequest2 = AllocateRequest.NewInstance(response2.GetResponseId(), 0F, null , null, null); response2 = Allocate(attempt2.GetAppAttemptId(), allocateRequest2); updatedNodes = response2.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(1, updatedNodes.Count); nr = updatedNodes.GetEnumerator().Next(); NUnit.Framework.Assert.AreEqual(nm4.GetNodeId(), nr.GetNodeId()); NUnit.Framework.Assert.AreEqual(NodeState.Running, nr.GetNodeState()); // subsequent allocate calls should return no updated nodes allocateRequest2 = AllocateRequest.NewInstance(response2.GetResponseId(), 0F, null , null, null); response2 = Allocate(attempt2.GetAppAttemptId(), allocateRequest2); updatedNodes = response2.GetUpdatedNodes(); NUnit.Framework.Assert.AreEqual(0, updatedNodes.Count); }