public virtual void TestNodeRegistrationVersionLessThanRM()
{
WriteToHostsFile("host2");
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
conf.Set(YarnConfiguration.RmNodemanagerMinimumVersion, "EqualToRM");
rm = new MockRM(conf);
rm.Start();
string nmVersion = "1.9.9";
ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService();
RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
>();
NodeId nodeId = NodeId.NewInstance("host2", 1234);
Resource capability = BuilderUtils.NewResource(1024, 1);
req.SetResource(capability);
req.SetNodeId(nodeId);
req.SetHttpPort(1234);
req.SetNMVersion(nmVersion);
// trying to register a invalid node.
RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager
(req);
NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response.GetNodeAction());
NUnit.Framework.Assert.IsTrue("Diagnostic message did not contain: 'Disallowed NodeManager "
+ "Version " + nmVersion + ", is less than the minimum version'", response.GetDiagnosticsMessage
().Contains("Disallowed NodeManager Version " + nmVersion + ", is less than the minimum version "
));
}
public virtual void TestSetRMIdentifierInRegistration()
{
Configuration conf = new Configuration();
rm = new MockRM(conf);
rm.Start();
MockNM nm = new MockNM("host1:1234", 5120, rm.GetResourceTrackerService());
RegisterNodeManagerResponse response = nm.RegisterNode();
// Verify the RMIdentifier is correctly set in RegisterNodeManagerResponse
NUnit.Framework.Assert.AreEqual(ResourceManager.GetClusterTimeStamp(), response.GetRMIdentifier
());
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual RegisterNodeManagerResponse RegisterNodeManager(RegisterNodeManagerRequest
request)
{
RegisterNodeManagerResponse response = recordFactory.NewRecordInstance <RegisterNodeManagerResponse
>();
MasterKey masterKey = new MasterKeyPBImpl();
masterKey.SetKeyId(123);
masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { 123 }));
response.SetContainerTokenMasterKey(masterKey);
response.SetNMTokenMasterKey(masterKey);
return(response);
}
/// <exception cref="System.Exception"/>
public static RegisterNodeManagerResponse SerDe(RegisterNodeManagerResponse orig)
{
RegisterNodeManagerResponsePBImpl asPB = (RegisterNodeManagerResponsePBImpl)orig;
YarnServerCommonServiceProtos.RegisterNodeManagerResponseProto proto = asPB.GetProto
();
ByteArrayOutputStream @out = new ByteArrayOutputStream();
proto.WriteTo(@out);
ByteArrayInputStream @in = new ByteArrayInputStream(@out.ToByteArray());
YarnServerCommonServiceProtos.RegisterNodeManagerResponseProto.Builder cp = YarnServerCommonServiceProtos.RegisterNodeManagerResponseProto
.NewBuilder();
cp.MergeFrom(@in);
return(new RegisterNodeManagerResponsePBImpl(((YarnServerCommonServiceProtos.RegisterNodeManagerResponseProto
)cp.Build())));
}
public virtual void TestRoundTrip()
{
RegisterNodeManagerResponse resp = recordFactory.NewRecordInstance <RegisterNodeManagerResponse
>();
byte[] b = new byte[] { 0, 1, 2, 3, 4, 5 };
MasterKey containerTokenMK = recordFactory.NewRecordInstance <MasterKey>();
containerTokenMK.SetKeyId(54321);
containerTokenMK.SetBytes(ByteBuffer.Wrap(b));
resp.SetContainerTokenMasterKey(containerTokenMK);
MasterKey nmTokenMK = recordFactory.NewRecordInstance <MasterKey>();
nmTokenMK.SetKeyId(12345);
nmTokenMK.SetBytes(ByteBuffer.Wrap(b));
resp.SetNMTokenMasterKey(nmTokenMK);
resp.SetNodeAction(NodeAction.Normal);
NUnit.Framework.Assert.AreEqual(NodeAction.Normal, resp.GetNodeAction());
// Verifying containerTokenMasterKey
NUnit.Framework.Assert.IsNotNull(resp.GetContainerTokenMasterKey());
NUnit.Framework.Assert.AreEqual(54321, resp.GetContainerTokenMasterKey().GetKeyId
());
Assert.AssertArrayEquals(b, ((byte[])resp.GetContainerTokenMasterKey().GetBytes()
.Array()));
RegisterNodeManagerResponse respCopy = SerDe(resp);
NUnit.Framework.Assert.AreEqual(NodeAction.Normal, respCopy.GetNodeAction());
NUnit.Framework.Assert.IsNotNull(respCopy.GetContainerTokenMasterKey());
NUnit.Framework.Assert.AreEqual(54321, respCopy.GetContainerTokenMasterKey().GetKeyId
());
Assert.AssertArrayEquals(b, ((byte[])respCopy.GetContainerTokenMasterKey().GetBytes
().Array()));
// Verifying nmTokenMasterKey
NUnit.Framework.Assert.IsNotNull(resp.GetNMTokenMasterKey());
NUnit.Framework.Assert.AreEqual(12345, resp.GetNMTokenMasterKey().GetKeyId());
Assert.AssertArrayEquals(b, ((byte[])resp.GetNMTokenMasterKey().GetBytes().Array(
)));
respCopy = SerDe(resp);
NUnit.Framework.Assert.AreEqual(NodeAction.Normal, respCopy.GetNodeAction());
NUnit.Framework.Assert.IsNotNull(respCopy.GetNMTokenMasterKey());
NUnit.Framework.Assert.AreEqual(12345, respCopy.GetNMTokenMasterKey().GetKeyId());
Assert.AssertArrayEquals(b, ((byte[])respCopy.GetNMTokenMasterKey().GetBytes().Array
()));
}
public virtual void TestNodeRegistrationWithMinimumAllocations()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmSchedulerMinimumAllocationMb, "2048");
conf.Set(YarnConfiguration.RmSchedulerMinimumAllocationVcores, "4");
rm = new MockRM(conf);
rm.Start();
ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService();
RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
>();
NodeId nodeId = BuilderUtils.NewNodeId("host", 1234);
req.SetNodeId(nodeId);
Resource capability = BuilderUtils.NewResource(1024, 1);
req.SetResource(capability);
RegisterNodeManagerResponse response1 = resourceTrackerService.RegisterNodeManager
(req);
NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response1.GetNodeAction());
capability.SetMemory(2048);
capability.SetVirtualCores(1);
req.SetResource(capability);
RegisterNodeManagerResponse response2 = resourceTrackerService.RegisterNodeManager
(req);
NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response2.GetNodeAction());
capability.SetMemory(1024);
capability.SetVirtualCores(4);
req.SetResource(capability);
RegisterNodeManagerResponse response3 = resourceTrackerService.RegisterNodeManager
(req);
NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response3.GetNodeAction());
capability.SetMemory(2048);
capability.SetVirtualCores(4);
req.SetResource(capability);
RegisterNodeManagerResponse response4 = resourceTrackerService.RegisterNodeManager
(req);
NUnit.Framework.Assert.AreEqual(NodeAction.Normal, response4.GetNodeAction());
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual YarnServerCommonServiceProtos.RegisterNodeManagerResponseProto RegisterNodeManager
(RpcController controller, YarnServerCommonServiceProtos.RegisterNodeManagerRequestProto
proto)
{
RegisterNodeManagerRequestPBImpl request = new RegisterNodeManagerRequestPBImpl(proto
);
try
{
RegisterNodeManagerResponse response = real.RegisterNodeManager(request);
return(((RegisterNodeManagerResponsePBImpl)response).GetProto());
}
catch (YarnException e)
{
throw new ServiceException(e);
}
catch (IOException e)
{
throw new ServiceException(e);
}
}
/// <exception cref="System.Exception"/>
public virtual RegisterNodeManagerResponse RegisterNode(IList <NMContainerStatus>
containerReports, IList <ApplicationId> runningApplications)
{
RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
>();
req.SetNodeId(nodeId);
req.SetHttpPort(httpPort);
Resource resource = BuilderUtils.NewResource(memory, vCores);
req.SetResource(resource);
req.SetContainerStatuses(containerReports);
req.SetNMVersion(version);
req.SetRunningApplications(runningApplications);
RegisterNodeManagerResponse registrationResponse = resourceTracker.RegisterNodeManager
(req);
this.currentContainerTokenMasterKey = registrationResponse.GetContainerTokenMasterKey
();
this.currentNMTokenMasterKey = registrationResponse.GetNMTokenMasterKey();
return(registrationResponse);
}
public virtual void TestNodeRegistrationFailure()
{
WriteToHostsFile("host1");
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
rm = new MockRM(conf);
rm.Start();
ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService();
RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
>();
NodeId nodeId = NodeId.NewInstance("host2", 1234);
req.SetNodeId(nodeId);
req.SetHttpPort(1234);
// trying to register a invalid node.
RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager
(req);
NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response.GetNodeAction());
NUnit.Framework.Assert.AreEqual("Disallowed NodeManager from host2, Sending SHUTDOWN signal to the NodeManager."
, response.GetDiagnosticsMessage());
}
public virtual void TestNodeRegistrationSuccess()
{
WriteToHostsFile("host2");
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
rm = new MockRM(conf);
rm.Start();
ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService();
RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
>();
NodeId nodeId = NodeId.NewInstance("host2", 1234);
Resource capability = BuilderUtils.NewResource(1024, 1);
req.SetResource(capability);
req.SetNodeId(nodeId);
req.SetHttpPort(1234);
req.SetNMVersion(YarnVersionInfo.GetVersion());
// trying to register a invalid node.
RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager
(req);
NUnit.Framework.Assert.AreEqual(NodeAction.Normal, response.GetNodeAction());
}
protected internal virtual void RegisterWithRM()
{
IList <NMContainerStatus> containerReports = GetNMContainerStatuses();
RegisterNodeManagerRequest request = RegisterNodeManagerRequest.NewInstance(nodeId
, httpPort, totalResource, nodeManagerVersionId, containerReports, GetRunningApplications
());
if (containerReports != null)
{
Log.Info("Registering with RM using containers :" + containerReports);
}
RegisterNodeManagerResponse regNMResponse = resourceTracker.RegisterNodeManager(request
);
this.rmIdentifier = regNMResponse.GetRMIdentifier();
// if the Resourcemanager instructs NM to shutdown.
if (NodeAction.Shutdown.Equals(regNMResponse.GetNodeAction()))
{
string message = "Message from ResourceManager: " + regNMResponse.GetDiagnosticsMessage
();
throw new YarnRuntimeException("Recieved SHUTDOWN signal from Resourcemanager ,Registration of NodeManager failed, "
+ message);
}
// if ResourceManager version is too old then shutdown
if (!minimumResourceManagerVersion.Equals("NONE"))
{
if (minimumResourceManagerVersion.Equals("EqualToNM"))
{
minimumResourceManagerVersion = nodeManagerVersionId;
}
string rmVersion = regNMResponse.GetRMVersion();
if (rmVersion == null)
{
string message = "The Resource Manager's did not return a version. " + "Valid version cannot be checked.";
throw new YarnRuntimeException("Shutting down the Node Manager. " + message);
}
if (VersionUtil.CompareVersions(rmVersion, minimumResourceManagerVersion) < 0)
{
string message = "The Resource Manager's version (" + rmVersion + ") is less than the minimum "
+ "allowed version " + minimumResourceManagerVersion;
throw new YarnRuntimeException("Shutting down the Node Manager on RM " + "version error, "
+ message);
}
}
MasterKey masterKey = regNMResponse.GetContainerTokenMasterKey();
// do this now so that its set before we start heartbeating to RM
// It is expected that status updater is started by this point and
// RM gives the shared secret in registration during
// StatusUpdater#start().
if (masterKey != null)
{
this.context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
}
masterKey = regNMResponse.GetNMTokenMasterKey();
if (masterKey != null)
{
this.context.GetNMTokenSecretManager().SetMasterKey(masterKey);
}
Log.Info("Registered with ResourceManager as " + this.nodeId + " with total resource of "
+ this.totalResource);
Log.Info("Notifying ContainerManager to unblock new container-requests");
((ContainerManagerImpl)this.context.GetContainerManager()).SetBlockNewContainerRequests
(false);
}
/// <exception cref="System.Exception"/>
private void ValidateRMNMKeyExchange(YarnConfiguration conf)
{
// Default rolling and activation intervals are large enough, no need to
// intervene
DrainDispatcher dispatcher = new DrainDispatcher();
ResourceManager rm = new _ResourceManager_56(dispatcher);
// Do nothing.
// Don't need it, skip.
rm.Init(conf);
rm.Start();
// Testing ContainerToken and NMToken
string containerToken = "Container Token : ";
string nmToken = "NM Token : ";
MockNM nm = new MockNM("host:1234", 3072, rm.GetResourceTrackerService());
RegisterNodeManagerResponse registrationResponse = nm.RegisterNode();
MasterKey containerTokenMasterKey = registrationResponse.GetContainerTokenMasterKey
();
NUnit.Framework.Assert.IsNotNull(containerToken + "Registration should cause a key-update!"
, containerTokenMasterKey);
MasterKey nmTokenMasterKey = registrationResponse.GetNMTokenMasterKey();
NUnit.Framework.Assert.IsNotNull(nmToken + "Registration should cause a key-update!"
, nmTokenMasterKey);
dispatcher.Await();
NodeHeartbeatResponse response = nm.NodeHeartbeat(true);
NUnit.Framework.Assert.IsNull(containerToken + "First heartbeat after registration shouldn't get any key updates!"
, response.GetContainerTokenMasterKey());
NUnit.Framework.Assert.IsNull(nmToken + "First heartbeat after registration shouldn't get any key updates!"
, response.GetNMTokenMasterKey());
dispatcher.Await();
response = nm.NodeHeartbeat(true);
NUnit.Framework.Assert.IsNull(containerToken + "Even second heartbeat after registration shouldn't get any key updates!"
, response.GetContainerTokenMasterKey());
NUnit.Framework.Assert.IsNull(nmToken + "Even second heartbeat after registration shouldn't get any key updates!"
, response.GetContainerTokenMasterKey());
dispatcher.Await();
// Let's force a roll-over
rm.GetRMContext().GetContainerTokenSecretManager().RollMasterKey();
rm.GetRMContext().GetNMTokenSecretManager().RollMasterKey();
// Heartbeats after roll-over and before activation should be fine.
response = nm.NodeHeartbeat(true);
NUnit.Framework.Assert.IsNotNull(containerToken + "Heartbeats after roll-over and before activation should not err out."
, response.GetContainerTokenMasterKey());
NUnit.Framework.Assert.IsNotNull(nmToken + "Heartbeats after roll-over and before activation should not err out."
, response.GetNMTokenMasterKey());
NUnit.Framework.Assert.AreEqual(containerToken + "Roll-over should have incremented the key-id only by one!"
, containerTokenMasterKey.GetKeyId() + 1, response.GetContainerTokenMasterKey().
GetKeyId());
NUnit.Framework.Assert.AreEqual(nmToken + "Roll-over should have incremented the key-id only by one!"
, nmTokenMasterKey.GetKeyId() + 1, response.GetNMTokenMasterKey().GetKeyId());
dispatcher.Await();
response = nm.NodeHeartbeat(true);
NUnit.Framework.Assert.IsNull(containerToken + "Second heartbeat after roll-over shouldn't get any key updates!"
, response.GetContainerTokenMasterKey());
NUnit.Framework.Assert.IsNull(nmToken + "Second heartbeat after roll-over shouldn't get any key updates!"
, response.GetNMTokenMasterKey());
dispatcher.Await();
// Let's force activation
rm.GetRMContext().GetContainerTokenSecretManager().ActivateNextMasterKey();
rm.GetRMContext().GetNMTokenSecretManager().ActivateNextMasterKey();
response = nm.NodeHeartbeat(true);
NUnit.Framework.Assert.IsNull(containerToken + "Activation shouldn't cause any key updates!"
, response.GetContainerTokenMasterKey());
NUnit.Framework.Assert.IsNull(nmToken + "Activation shouldn't cause any key updates!"
, response.GetNMTokenMasterKey());
dispatcher.Await();
response = nm.NodeHeartbeat(true);
NUnit.Framework.Assert.IsNull(containerToken + "Even second heartbeat after activation shouldn't get any key updates!"
, response.GetContainerTokenMasterKey());
NUnit.Framework.Assert.IsNull(nmToken + "Even second heartbeat after activation shouldn't get any key updates!"
, response.GetNMTokenMasterKey());
dispatcher.Await();
rm.Stop();
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual RegisterNodeManagerResponse RegisterNodeManager(RegisterNodeManagerRequest
request)
{
NodeId nodeId = request.GetNodeId();
string host = nodeId.GetHost();
int cmPort = nodeId.GetPort();
int httpPort = request.GetHttpPort();
Resource capability = request.GetResource();
string nodeManagerVersion = request.GetNMVersion();
RegisterNodeManagerResponse response = recordFactory.NewRecordInstance <RegisterNodeManagerResponse
>();
if (!minimumNodeManagerVersion.Equals("NONE"))
{
if (minimumNodeManagerVersion.Equals("EqualToRM"))
{
minimumNodeManagerVersion = YarnVersionInfo.GetVersion();
}
if ((nodeManagerVersion == null) || (VersionUtil.CompareVersions(nodeManagerVersion
, minimumNodeManagerVersion)) < 0)
{
string message = "Disallowed NodeManager Version " + nodeManagerVersion + ", is less than the minimum version "
+ minimumNodeManagerVersion + " sending SHUTDOWN signal to " + "NodeManager.";
Log.Info(message);
response.SetDiagnosticsMessage(message);
response.SetNodeAction(NodeAction.Shutdown);
return(response);
}
}
// Check if this node is a 'valid' node
if (!this.nodesListManager.IsValidNode(host))
{
string message = "Disallowed NodeManager from " + host + ", Sending SHUTDOWN signal to the NodeManager.";
Log.Info(message);
response.SetDiagnosticsMessage(message);
response.SetNodeAction(NodeAction.Shutdown);
return(response);
}
// Check if this node has minimum allocations
if (capability.GetMemory() < minAllocMb || capability.GetVirtualCores() < minAllocVcores)
{
string message = "NodeManager from " + host + " doesn't satisfy minimum allocations, Sending SHUTDOWN"
+ " signal to the NodeManager.";
Log.Info(message);
response.SetDiagnosticsMessage(message);
response.SetNodeAction(NodeAction.Shutdown);
return(response);
}
response.SetContainerTokenMasterKey(containerTokenSecretManager.GetCurrentKey());
response.SetNMTokenMasterKey(nmTokenSecretManager.GetCurrentKey());
RMNode rmNode = new RMNodeImpl(nodeId, rmContext, host, cmPort, httpPort, Resolve
(host), capability, nodeManagerVersion);
RMNode oldNode = this.rmContext.GetRMNodes().PutIfAbsent(nodeId, rmNode);
if (oldNode == null)
{
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeStartedEvent(nodeId
, request.GetNMContainerStatuses(), request.GetRunningApplications()));
}
else
{
Log.Info("Reconnect from the node at: " + host);
this.nmLivelinessMonitor.Unregister(nodeId);
// Reset heartbeat ID since node just restarted.
oldNode.ResetLastNodeHeartBeatResponse();
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeReconnectEvent(
nodeId, rmNode, request.GetRunningApplications(), request.GetNMContainerStatuses
()));
}
// On every node manager register we will be clearing NMToken keys if
// present for any running application.
this.nmTokenSecretManager.RemoveNodeKey(nodeId);
this.nmLivelinessMonitor.Register(nodeId);
// Handle received container status, this should be processed after new
// RMNode inserted
if (!rmContext.IsWorkPreservingRecoveryEnabled())
{
if (!request.GetNMContainerStatuses().IsEmpty())
{
Log.Info("received container statuses on node manager register :" + request.GetNMContainerStatuses
());
foreach (NMContainerStatus status in request.GetNMContainerStatuses())
{
HandleNMContainerStatus(status, nodeId);
}
}
}
string message_1 = "NodeManager from node " + host + "(cmPort: " + cmPort + " httpPort: "
+ httpPort + ") " + "registered with capability: " + capability + ", assigned nodeId "
+ nodeId;
Log.Info(message_1);
response.SetNodeAction(NodeAction.Normal);
response.SetRMIdentifier(ResourceManager.GetClusterTimeStamp());
response.SetRMVersion(YarnVersionInfo.GetVersion());
return(response);
}
