private void SetHealthStatus(NodeHealthStatus healthStatus, bool isHealthy, string
healthReport, long lastHealthReportTime)
{
healthStatus.SetHealthReport(healthReport);
healthStatus.SetIsNodeHealthy(isHealthy);
healthStatus.SetLastHealthReportTime(lastHealthReportTime);
}
private NodeHealthStatus GetNodeHealthStatus()
{
NodeHealthStatus healStatus = recordFactory.NewRecordInstance <NodeHealthStatus>();
healStatus.SetHealthReport("healthReport");
healStatus.SetIsNodeHealthy(true);
healStatus.SetLastHealthReportTime(1000);
return(healStatus);
}
public RMNodeStatusEvent(NodeId nodeId, NodeHealthStatus nodeHealthStatus, IList <
ContainerStatus> collection, IList <ApplicationId> keepAliveAppIds, NodeHeartbeatResponse
latestResponse)
: base(nodeId, RMNodeEventType.StatusUpdate)
{
this.nodeHealthStatus = nodeHealthStatus;
this.containersCollection = collection;
this.keepAliveAppIds = keepAliveAppIds;
this.latestResponse = latestResponse;
}
private RMNodeImpl GetUnhealthyNode()
{
RMNodeImpl node = GetRunningNode();
NodeHealthStatus status = NodeHealthStatus.NewInstance(false, "sick", Runtime.CurrentTimeMillis
());
node.Handle(new RMNodeStatusEvent(node.GetNodeID(), status, new AList <ContainerStatus
>(), null, null));
NUnit.Framework.Assert.AreEqual(NodeState.Unhealthy, node.GetState());
return(node);
}
public override void SetNodeHealthStatus(NodeHealthStatus healthStatus)
{
lock (this)
{
MaybeInitBuilder();
if (healthStatus == null)
{
builder.ClearNodeHealthStatus();
}
this.nodeHealthStatus = healthStatus;
}
}
public static NodeStatus CreateNodeStatus(NodeId nodeId, IList <ContainerStatus> containers
)
{
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
NodeStatus nodeStatus = recordFactory.NewRecordInstance <NodeStatus>();
nodeStatus.SetNodeId(nodeId);
nodeStatus.SetContainersStatuses(containers);
NodeHealthStatus nodeHealthStatus = recordFactory.NewRecordInstance <NodeHealthStatus
>();
nodeHealthStatus.SetIsNodeHealthy(true);
nodeStatus.SetNodeHealthStatus(nodeHealthStatus);
return(nodeStatus);
}
private RMNodeStatusEvent GetMockRMNodeStatusEvent()
{
NodeHeartbeatResponse response = Org.Mockito.Mockito.Mock <NodeHeartbeatResponse>(
);
NodeHealthStatus healthStatus = Org.Mockito.Mockito.Mock <NodeHealthStatus>();
bool yes = true;
Org.Mockito.Mockito.DoReturn(yes).When(healthStatus).GetIsNodeHealthy();
RMNodeStatusEvent @event = Org.Mockito.Mockito.Mock <RMNodeStatusEvent>();
Org.Mockito.Mockito.DoReturn(healthStatus).When(@event).GetNodeHealthStatus();
Org.Mockito.Mockito.DoReturn(response).When(@event).GetLatestResponse();
Org.Mockito.Mockito.DoReturn(RMNodeEventType.StatusUpdate).When(@event).GetType();
return(@event);
}
public virtual void TestRPCResponseId()
{
string node = "localhost";
Resource capability = BuilderUtils.NewResource(1024, 1);
RegisterNodeManagerRequest request = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
nodeId = NodeId.NewInstance(node, 1234);
request.SetNodeId(nodeId);
request.SetHttpPort(0);
request.SetResource(capability);
RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
request1.SetNodeId(nodeId);
request1.SetHttpPort(0);
request1.SetResource(capability);
resourceTrackerService.RegisterNodeManager(request1);
NodeStatus nodeStatus = recordFactory.NewRecordInstance <NodeStatus>();
nodeStatus.SetNodeId(nodeId);
NodeHealthStatus nodeHealthStatus = recordFactory.NewRecordInstance <NodeHealthStatus
>();
nodeHealthStatus.SetIsNodeHealthy(true);
nodeStatus.SetNodeHealthStatus(nodeHealthStatus);
NodeHeartbeatRequest nodeHeartBeatRequest = recordFactory.NewRecordInstance <NodeHeartbeatRequest
>();
nodeHeartBeatRequest.SetNodeStatus(nodeStatus);
nodeStatus.SetResponseId(0);
NodeHeartbeatResponse response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest
);
NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 1);
nodeStatus.SetResponseId(response.GetResponseId());
response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 2);
/* try calling with less response id */
response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 2);
nodeStatus.SetResponseId(0);
response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
NUnit.Framework.Assert.IsTrue(NodeAction.Resync.Equals(response.GetNodeAction()));
NUnit.Framework.Assert.AreEqual("Too far behind rm response id:2 nm response id:0"
, response.GetDiagnosticsMessage());
}
public override NodeHealthStatus GetNodeHealthStatus()
{
lock (this)
{
YarnServerCommonProtos.NodeStatusProtoOrBuilder p = viaProto ? proto : builder;
if (nodeHealthStatus != null)
{
return(nodeHealthStatus);
}
if (!p.HasNodeHealthStatus())
{
return(null);
}
nodeHealthStatus = ConvertFromProtoFormat(p.GetNodeHealthStatus());
return(nodeHealthStatus);
}
}
/// <exception cref="System.IO.IOException"/>
private NodeStatus GetNodeStatus(int responseId)
{
NodeHealthStatus nodeHealthStatus = this.context.GetNodeHealthStatus();
nodeHealthStatus.SetHealthReport(healthChecker.GetHealthReport());
nodeHealthStatus.SetIsNodeHealthy(healthChecker.IsHealthy());
nodeHealthStatus.SetLastHealthReportTime(healthChecker.GetLastHealthReportTime());
if (Log.IsDebugEnabled())
{
Log.Debug("Node's health-status : " + nodeHealthStatus.GetIsNodeHealthy() + ", "
+ nodeHealthStatus.GetHealthReport());
}
IList <ContainerStatus> containersStatuses = GetContainerStatuses();
NodeStatus nodeStatus = NodeStatus.NewInstance(nodeId, responseId, containersStatuses
, CreateKeepAliveApplicationList(), nodeHealthStatus);
return(nodeStatus);
}
/// <exception cref="System.Exception"/>
public virtual NodeHeartbeatResponse NodeHeartbeat(IDictionary <ApplicationId, IList
<Org.Apache.Hadoop.Yarn.Api.Records.ContainerStatus> > conts, bool isHealthy, int
resId)
{
NodeHeartbeatRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NodeHeartbeatRequest
>();
NodeStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NodeStatus>();
status.SetResponseId(resId);
status.SetNodeId(nodeId);
foreach (KeyValuePair <ApplicationId, IList <Org.Apache.Hadoop.Yarn.Api.Records.ContainerStatus
> > entry in conts)
{
Org.Mortbay.Log.Log.Info("entry.getValue() " + entry.Value);
status.SetContainersStatuses(entry.Value);
}
NodeHealthStatus healthStatus = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NodeHealthStatus
>();
healthStatus.SetHealthReport(string.Empty);
healthStatus.SetIsNodeHealthy(isHealthy);
healthStatus.SetLastHealthReportTime(1);
status.SetNodeHealthStatus(healthStatus);
req.SetNodeStatus(status);
req.SetLastKnownContainerTokenMasterKey(this.currentContainerTokenMasterKey);
req.SetLastKnownNMTokenMasterKey(this.currentNMTokenMasterKey);
NodeHeartbeatResponse heartbeatResponse = resourceTracker.NodeHeartbeat(req);
MasterKey masterKeyFromRM = heartbeatResponse.GetContainerTokenMasterKey();
if (masterKeyFromRM != null && masterKeyFromRM.GetKeyId() != this.currentContainerTokenMasterKey
.GetKeyId())
{
this.currentContainerTokenMasterKey = masterKeyFromRM;
}
masterKeyFromRM = heartbeatResponse.GetNMTokenMasterKey();
if (masterKeyFromRM != null && masterKeyFromRM.GetKeyId() != this.currentNMTokenMasterKey
.GetKeyId())
{
this.currentNMTokenMasterKey = masterKeyFromRM;
}
return(heartbeatResponse);
}
public virtual void TestNodesDefaultWithUnHealthyNode()
{
WebResource r = Resource();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
MockNM nm2 = rm.RegisterNode("h2:1235", 5121);
rm.SendNodeStarted(nm1);
rm.NMwaitForState(nm1.GetNodeId(), NodeState.Running);
rm.NMwaitForState(nm2.GetNodeId(), NodeState.New);
MockNM nm3 = rm.RegisterNode("h3:1236", 5122);
rm.NMwaitForState(nm3.GetNodeId(), NodeState.New);
rm.SendNodeStarted(nm3);
rm.NMwaitForState(nm3.GetNodeId(), NodeState.Running);
RMNodeImpl node = (RMNodeImpl)rm.GetRMContext().GetRMNodes()[nm3.GetNodeId()];
NodeHealthStatus nodeHealth = NodeHealthStatus.NewInstance(false, "test health report"
, Runtime.CurrentTimeMillis());
node.Handle(new RMNodeStatusEvent(nm3.GetNodeId(), nodeHealth, new AList <ContainerStatus
>(), null, null));
rm.NMwaitForState(nm3.GetNodeId(), NodeState.Unhealthy);
ClientResponse response = r.Path("ws").Path("v1").Path("cluster").Path("nodes").Accept
(MediaType.ApplicationJson).Get <ClientResponse>();
NUnit.Framework.Assert.AreEqual(MediaType.ApplicationJsonType, response.GetType()
);
JSONObject json = response.GetEntity <JSONObject>();
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, json.Length());
JSONObject nodes = json.GetJSONObject("nodes");
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 1, nodes.Length()
);
JSONArray nodeArray = nodes.GetJSONArray("node");
// 3 nodes, including the unhealthy node and the new node.
NUnit.Framework.Assert.AreEqual("incorrect number of elements", 3, nodeArray.Length
());
}
public virtual void TestNodeHealthScript()
{
RecordFactory factory = RecordFactoryProvider.GetRecordFactory(null);
NodeHealthStatus healthStatus = factory.NewRecordInstance <NodeHealthStatus>();
string errorScript = "echo ERROR\n echo \"Tracker not healthy\"";
string normalScript = "echo \"I am all fine\"";
string timeOutScript = Shell.Windows ? "@echo off\nping -n 4 127.0.0.1 >nul\necho \"I am fine\""
: "sleep 4\necho \"I am fine\"";
Configuration conf = GetConfForNodeHealthScript();
conf.WriteXml(new FileOutputStream(nodeHealthConfigFile));
conf.AddResource(nodeHealthConfigFile.GetName());
WriteNodeHealthScriptFile(normalScript, true);
NodeHealthCheckerService nodeHealthChecker = new NodeHealthCheckerService();
nodeHealthChecker.Init(conf);
NodeHealthScriptRunner nodeHealthScriptRunner = nodeHealthChecker.GetNodeHealthScriptRunner
();
TimerTask timerTask = nodeHealthScriptRunner.GetTimerTask();
timerTask.Run();
SetHealthStatus(healthStatus, nodeHealthChecker.IsHealthy(), nodeHealthChecker.GetHealthReport
(), nodeHealthChecker.GetLastHealthReportTime());
Log.Info("Checking initial healthy condition");
// Check proper report conditions.
NUnit.Framework.Assert.IsTrue("Node health status reported unhealthy", healthStatus
.GetIsNodeHealthy());
NUnit.Framework.Assert.IsTrue("Node health status reported unhealthy", healthStatus
.GetHealthReport().Equals(nodeHealthChecker.GetHealthReport()));
// write out error file.
// Healthy to unhealthy transition
WriteNodeHealthScriptFile(errorScript, true);
// Run timer
timerTask.Run();
// update health status
SetHealthStatus(healthStatus, nodeHealthChecker.IsHealthy(), nodeHealthChecker.GetHealthReport
(), nodeHealthChecker.GetLastHealthReportTime());
Log.Info("Checking Healthy--->Unhealthy");
NUnit.Framework.Assert.IsFalse("Node health status reported healthy", healthStatus
.GetIsNodeHealthy());
NUnit.Framework.Assert.IsTrue("Node health status reported healthy", healthStatus
.GetHealthReport().Equals(nodeHealthChecker.GetHealthReport()));
// Check unhealthy to healthy transitions.
WriteNodeHealthScriptFile(normalScript, true);
timerTask.Run();
SetHealthStatus(healthStatus, nodeHealthChecker.IsHealthy(), nodeHealthChecker.GetHealthReport
(), nodeHealthChecker.GetLastHealthReportTime());
Log.Info("Checking UnHealthy--->healthy");
// Check proper report conditions.
NUnit.Framework.Assert.IsTrue("Node health status reported unhealthy", healthStatus
.GetIsNodeHealthy());
NUnit.Framework.Assert.IsTrue("Node health status reported unhealthy", healthStatus
.GetHealthReport().Equals(nodeHealthChecker.GetHealthReport()));
// Healthy to timeout transition.
WriteNodeHealthScriptFile(timeOutScript, true);
timerTask.Run();
SetHealthStatus(healthStatus, nodeHealthChecker.IsHealthy(), nodeHealthChecker.GetHealthReport
(), nodeHealthChecker.GetLastHealthReportTime());
Log.Info("Checking Healthy--->timeout");
NUnit.Framework.Assert.IsFalse("Node health status reported healthy even after timeout"
, healthStatus.GetIsNodeHealthy());
NUnit.Framework.Assert.IsTrue("Node script time out message not propogated", healthStatus
.GetHealthReport().Equals(NodeHealthScriptRunner.NodeHealthScriptTimedOutMsg + NodeHealthCheckerService
.Separator + nodeHealthChecker.GetDiskHandler().GetDisksHealthReport(false)));
}
private YarnServerCommonProtos.NodeHealthStatusProto ConvertToProtoFormat(NodeHealthStatus
healthStatus)
{
return(((NodeHealthStatusPBImpl)healthStatus).GetProto());
}