public virtual void VerifyClusterGeneric(long clusterid, long startedon, string state
, string haState, string haZooKeeperConnectionState, string hadoopVersionBuiltOn
, string hadoopBuildVersion, string hadoopVersion, string resourceManagerVersionBuiltOn
, string resourceManagerBuildVersion, string resourceManagerVersion)
{
NUnit.Framework.Assert.AreEqual("clusterId doesn't match: ", ResourceManager.GetClusterTimeStamp
(), clusterid);
NUnit.Framework.Assert.AreEqual("startedOn doesn't match: ", ResourceManager.GetClusterTimeStamp
(), startedon);
NUnit.Framework.Assert.IsTrue("stated doesn't match: " + state, state.Matches(Service.STATE
.Inited.ToString()));
NUnit.Framework.Assert.IsTrue("HA state doesn't match: " + haState, haState.Matches
("INITIALIZING"));
WebServicesTestUtils.CheckStringMatch("hadoopVersionBuiltOn", VersionInfo.GetDate
(), hadoopVersionBuiltOn);
WebServicesTestUtils.CheckStringEqual("hadoopBuildVersion", VersionInfo.GetBuildVersion
(), hadoopBuildVersion);
WebServicesTestUtils.CheckStringMatch("hadoopVersion", VersionInfo.GetVersion(),
hadoopVersion);
WebServicesTestUtils.CheckStringMatch("resourceManagerVersionBuiltOn", YarnVersionInfo
.GetDate(), resourceManagerVersionBuiltOn);
WebServicesTestUtils.CheckStringEqual("resourceManagerBuildVersion", YarnVersionInfo
.GetBuildVersion(), resourceManagerBuildVersion);
WebServicesTestUtils.CheckStringMatch("resourceManagerVersion", YarnVersionInfo.GetVersion
(), resourceManagerVersion);
}
public ClusterInfo(ResourceManager rm)
{
// JAXB needs this
long ts = ResourceManager.GetClusterTimeStamp();
this.id = ts;
this.state = rm.GetServiceState();
this.haState = rm.GetRMContext().GetHAServiceState();
this.rmStateStoreName = rm.GetRMContext().GetStateStore().GetType().FullName;
this.startedOn = ts;
this.resourceManagerVersion = YarnVersionInfo.GetVersion();
this.resourceManagerBuildVersion = YarnVersionInfo.GetBuildVersion();
this.resourceManagerVersionBuiltOn = YarnVersionInfo.GetDate();
this.hadoopVersion = VersionInfo.GetVersion();
this.hadoopBuildVersion = VersionInfo.GetBuildVersion();
this.hadoopVersionBuiltOn = VersionInfo.GetDate();
this.haZooKeeperConnectionState = rm.GetRMContext().GetRMAdminService().GetHAZookeeperConnectionState
();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public NodeManager(string hostName, int containerManagerPort, int httpPort, string
rackName, Resource capability, ResourceManager resourceManager)
{
this.containerManagerAddress = hostName + ":" + containerManagerPort;
this.nodeHttpAddress = hostName + ":" + httpPort;
this.rackName = rackName;
this.resourceTrackerService = resourceManager.GetResourceTrackerService();
this.capability = capability;
Resources.AddTo(available, capability);
this.nodeId = NodeId.NewInstance(hostName, containerManagerPort);
RegisterNodeManagerRequest request = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
request.SetHttpPort(httpPort);
request.SetResource(capability);
request.SetNodeId(this.nodeId);
request.SetNMVersion(YarnVersionInfo.GetVersion());
resourceTrackerService.RegisterNodeManager(request);
this.resourceManager = resourceManager;
resourceManager.GetResourceScheduler().GetNodeReport(this.nodeId);
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
// NodeManager is the last service to start, so NodeId is available.
this.nodeId = this.context.GetNodeId();
this.httpPort = this.context.GetHttpPort();
this.nodeManagerVersionId = YarnVersionInfo.GetVersion();
try
{
// Registration has to be in start so that ContainerManager can get the
// perNM tokens needed to authenticate ContainerTokens.
this.resourceTracker = GetRMClient();
RegisterWithRM();
base.ServiceStart();
StartStatusUpdater();
}
catch (Exception e)
{
string errorMessage = "Unexpected error starting NodeStatusUpdater";
Log.Error(errorMessage, e);
throw new YarnRuntimeException(e);
}
}
public NodeInfo(Context context, ResourceView resourceView)
{
// JAXB needs this
this.id = context.GetNodeId().ToString();
this.nodeHostName = context.GetNodeId().GetHost();
this.totalVmemAllocatedContainersMB = resourceView.GetVmemAllocatedForContainers(
) / BytesInMb;
this.vmemCheckEnabled = resourceView.IsVmemCheckEnabled();
this.totalPmemAllocatedContainersMB = resourceView.GetPmemAllocatedForContainers(
) / BytesInMb;
this.pmemCheckEnabled = resourceView.IsPmemCheckEnabled();
this.totalVCoresAllocatedContainers = resourceView.GetVCoresAllocatedForContainers
();
this.nodeHealthy = context.GetNodeHealthStatus().GetIsNodeHealthy();
this.lastNodeUpdateTime = context.GetNodeHealthStatus().GetLastHealthReportTime();
this.healthReport = context.GetNodeHealthStatus().GetHealthReport();
this.nodeManagerVersion = YarnVersionInfo.GetVersion();
this.nodeManagerBuildVersion = YarnVersionInfo.GetBuildVersion();
this.nodeManagerVersionBuiltOn = YarnVersionInfo.GetDate();
this.hadoopVersion = VersionInfo.GetVersion();
this.hadoopBuildVersion = VersionInfo.GetBuildVersion();
this.hadoopVersionBuiltOn = VersionInfo.GetDate();
}
public virtual void VerifyNodeInfoGeneric(string id, string healthReport, long totalVmemAllocatedContainersMB
, long totalPmemAllocatedContainersMB, long totalVCoresAllocatedContainers, bool
vmemCheckEnabled, bool pmemCheckEnabled, long lastNodeUpdateTime, bool nodeHealthy
, string nodeHostName, string hadoopVersionBuiltOn, string hadoopBuildVersion, string
hadoopVersion, string resourceManagerVersionBuiltOn, string resourceManagerBuildVersion
, string resourceManagerVersion)
{
WebServicesTestUtils.CheckStringMatch("id", "testhost.foo.com:8042", id);
WebServicesTestUtils.CheckStringMatch("healthReport", "Healthy", healthReport);
NUnit.Framework.Assert.AreEqual("totalVmemAllocatedContainersMB incorrect", 15872
, totalVmemAllocatedContainersMB);
NUnit.Framework.Assert.AreEqual("totalPmemAllocatedContainersMB incorrect", 16384
, totalPmemAllocatedContainersMB);
NUnit.Framework.Assert.AreEqual("totalVCoresAllocatedContainers incorrect", 4000,
totalVCoresAllocatedContainers);
NUnit.Framework.Assert.AreEqual("vmemCheckEnabled incorrect", true, vmemCheckEnabled
);
NUnit.Framework.Assert.AreEqual("pmemCheckEnabled incorrect", true, pmemCheckEnabled
);
NUnit.Framework.Assert.IsTrue("lastNodeUpdateTime incorrect", lastNodeUpdateTime
== nmContext.GetNodeHealthStatus().GetLastHealthReportTime());
NUnit.Framework.Assert.IsTrue("nodeHealthy isn't true", nodeHealthy);
WebServicesTestUtils.CheckStringMatch("nodeHostName", "testhost.foo.com", nodeHostName
);
WebServicesTestUtils.CheckStringMatch("hadoopVersionBuiltOn", VersionInfo.GetDate
(), hadoopVersionBuiltOn);
WebServicesTestUtils.CheckStringEqual("hadoopBuildVersion", VersionInfo.GetBuildVersion
(), hadoopBuildVersion);
WebServicesTestUtils.CheckStringMatch("hadoopVersion", VersionInfo.GetVersion(),
hadoopVersion);
WebServicesTestUtils.CheckStringMatch("resourceManagerVersionBuiltOn", YarnVersionInfo
.GetDate(), resourceManagerVersionBuiltOn);
WebServicesTestUtils.CheckStringEqual("resourceManagerBuildVersion", YarnVersionInfo
.GetBuildVersion(), resourceManagerBuildVersion);
WebServicesTestUtils.CheckStringMatch("resourceManagerVersion", YarnVersionInfo.GetVersion
(), resourceManagerVersion);
}
public virtual void TestNodeRegistrationSuccess()
{
WriteToHostsFile("host2");
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
rm = new MockRM(conf);
rm.Start();
ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService();
RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
>();
NodeId nodeId = NodeId.NewInstance("host2", 1234);
Resource capability = BuilderUtils.NewResource(1024, 1);
req.SetResource(capability);
req.SetNodeId(nodeId);
req.SetHttpPort(1234);
req.SetNMVersion(YarnVersionInfo.GetVersion());
// trying to register a invalid node.
RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager
(req);
NUnit.Framework.Assert.AreEqual(NodeAction.Normal, response.GetNodeAction());
}
/// <exception cref="System.Exception"/>
public virtual void TestResourceTrackerOnHA()
{
NodeId nodeId = NodeId.NewInstance("localhost", 0);
Resource resource = Resource.NewInstance(2048, 4);
// make sure registerNodeManager works when failover happens
RegisterNodeManagerRequest request = RegisterNodeManagerRequest.NewInstance(nodeId
, 0, resource, YarnVersionInfo.GetVersion(), null, null);
resourceTracker.RegisterNodeManager(request);
NUnit.Framework.Assert.IsTrue(WaitForNodeManagerToConnect(10000, nodeId));
// restart the failover thread, and make sure nodeHeartbeat works
failoverThread = CreateAndStartFailoverThread();
NodeStatus status = NodeStatus.NewInstance(NodeId.NewInstance("localhost", 0), 0,
null, null, null);
NodeHeartbeatRequest request2 = NodeHeartbeatRequest.NewInstance(status, null, null
);
resourceTracker.NodeHeartbeat(request2);
}
public MockNM(string nodeIdStr, int memory, int vcores, ResourceTrackerService resourceTracker
)
: this(nodeIdStr, memory, vcores, resourceTracker, YarnVersionInfo.GetVersion())
{
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual RegisterNodeManagerResponse RegisterNodeManager(RegisterNodeManagerRequest
request)
{
NodeId nodeId = request.GetNodeId();
string host = nodeId.GetHost();
int cmPort = nodeId.GetPort();
int httpPort = request.GetHttpPort();
Resource capability = request.GetResource();
string nodeManagerVersion = request.GetNMVersion();
RegisterNodeManagerResponse response = recordFactory.NewRecordInstance <RegisterNodeManagerResponse
>();
if (!minimumNodeManagerVersion.Equals("NONE"))
{
if (minimumNodeManagerVersion.Equals("EqualToRM"))
{
minimumNodeManagerVersion = YarnVersionInfo.GetVersion();
}
if ((nodeManagerVersion == null) || (VersionUtil.CompareVersions(nodeManagerVersion
, minimumNodeManagerVersion)) < 0)
{
string message = "Disallowed NodeManager Version " + nodeManagerVersion + ", is less than the minimum version "
+ minimumNodeManagerVersion + " sending SHUTDOWN signal to " + "NodeManager.";
Log.Info(message);
response.SetDiagnosticsMessage(message);
response.SetNodeAction(NodeAction.Shutdown);
return(response);
}
}
// Check if this node is a 'valid' node
if (!this.nodesListManager.IsValidNode(host))
{
string message = "Disallowed NodeManager from " + host + ", Sending SHUTDOWN signal to the NodeManager.";
Log.Info(message);
response.SetDiagnosticsMessage(message);
response.SetNodeAction(NodeAction.Shutdown);
return(response);
}
// Check if this node has minimum allocations
if (capability.GetMemory() < minAllocMb || capability.GetVirtualCores() < minAllocVcores)
{
string message = "NodeManager from " + host + " doesn't satisfy minimum allocations, Sending SHUTDOWN"
+ " signal to the NodeManager.";
Log.Info(message);
response.SetDiagnosticsMessage(message);
response.SetNodeAction(NodeAction.Shutdown);
return(response);
}
response.SetContainerTokenMasterKey(containerTokenSecretManager.GetCurrentKey());
response.SetNMTokenMasterKey(nmTokenSecretManager.GetCurrentKey());
RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort, Resolve
(host), capability, nodeManagerVersion);
RMNode oldNode = this.rmContext.GetRMNodes().PutIfAbsent(nodeId, rmNode);
if (oldNode == null)
{
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeStartedEvent(nodeId
, request.GetNMContainerStatuses(), request.GetRunningApplications()));
}
else
{
Log.Info("Reconnect from the node at: " + host);
this.nmLivelinessMonitor.Unregister(nodeId);
// Reset heartbeat ID since node just restarted.
oldNode.ResetLastNodeHeartBeatResponse();
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeReconnectEvent(
nodeId, rmNode, request.GetRunningApplications(), request.GetNMContainerStatuses
()));
}
// On every node manager register we will be clearing NMToken keys if
// present for any running application.
this.nmTokenSecretManager.RemoveNodeKey(nodeId);
this.nmLivelinessMonitor.Register(nodeId);
// Handle received container status, this should be processed after new
// RMNode inserted
if (!rmContext.IsWorkPreservingRecoveryEnabled())
{
if (!request.GetNMContainerStatuses().IsEmpty())
{
Log.Info("received container statuses on node manager register :" + request.GetNMContainerStatuses
());
foreach (NMContainerStatus status in request.GetNMContainerStatuses())
{
HandleNMContainerStatus(status, nodeId);
}
}
}
string message_1 = "NodeManager from node " + host + "(cmPort: " + cmPort + " httpPort: "
+ httpPort + ") " + "registered with capability: " + capability + ", assigned nodeId "
+ nodeId;
Log.Info(message_1);
response.SetNodeAction(NodeAction.Normal);
response.SetRMIdentifier(ResourceManager.GetClusterTimeStamp());
response.SetRMVersion(YarnVersionInfo.GetVersion());
return(response);
}
/// <exception cref="System.Exception"/>
public virtual MockNM RegisterNode(string nodeIdStr, int memory, int vCores, IList
<ApplicationId> runningApplications)
{
MockNM nm = new MockNM(nodeIdStr, memory, vCores, GetResourceTrackerService(), YarnVersionInfo
.GetVersion());
nm.RegisterNode(runningApplications);
return(nm);
}
