public virtual void TestAddNewExcludePathToConfiguration()
{
Configuration conf = new Configuration();
rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
MockNM nm2 = rm.RegisterNode("host2:5678", 10240);
ClusterMetrics metrics = ClusterMetrics.GetMetrics();
System.Diagnostics.Debug.Assert((metrics != null));
int initialMetricCount = metrics.GetNumDecommisionedNMs();
NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual(NodeAction.Normal, nodeHeartbeat.GetNodeAction());
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual(NodeAction.Normal, nodeHeartbeat.GetNodeAction());
WriteToHostsFile("host2");
conf.Set(YarnConfiguration.RmNodesExcludeFilePath, hostFile.GetAbsolutePath());
rm.GetNodesListManager().RefreshNodes(conf);
CheckDecommissionedNMCount(rm, ++initialMetricCount);
nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual("Node should not have been decomissioned.", NodeAction
.Normal, nodeHeartbeat.GetNodeAction());
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual("Node should have been decomissioned but is in state"
+ nodeHeartbeat.GetNodeAction(), NodeAction.Shutdown, nodeHeartbeat.GetNodeAction
());
}
public virtual void TestReboot()
{
Configuration conf = new Configuration();
rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
MockNM nm2 = rm.RegisterNode("host2:1234", 2048);
int initialMetricCount = ClusterMetrics.GetMetrics().GetNumRebootedNMs();
NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
nodeHeartbeat = nm2.NodeHeartbeat(new Dictionary <ApplicationId, IList <ContainerStatus
> >(), true, -100);
NUnit.Framework.Assert.IsTrue(NodeAction.Resync.Equals(nodeHeartbeat.GetNodeAction
()));
NUnit.Framework.Assert.AreEqual("Too far behind rm response id:0 nm response id:-100"
, nodeHeartbeat.GetDiagnosticsMessage());
CheckRebootedNMCount(rm, ++initialMetricCount);
}
public virtual void TestRPCResponseId()
{
string node = "localhost";
Resource capability = BuilderUtils.NewResource(1024, 1);
RegisterNodeManagerRequest request = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
nodeId = NodeId.NewInstance(node, 1234);
request.SetNodeId(nodeId);
request.SetHttpPort(0);
request.SetResource(capability);
RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
request1.SetNodeId(nodeId);
request1.SetHttpPort(0);
request1.SetResource(capability);
resourceTrackerService.RegisterNodeManager(request1);
NodeStatus nodeStatus = recordFactory.NewRecordInstance <NodeStatus>();
nodeStatus.SetNodeId(nodeId);
NodeHealthStatus nodeHealthStatus = recordFactory.NewRecordInstance <NodeHealthStatus
>();
nodeHealthStatus.SetIsNodeHealthy(true);
nodeStatus.SetNodeHealthStatus(nodeHealthStatus);
NodeHeartbeatRequest nodeHeartBeatRequest = recordFactory.NewRecordInstance <NodeHeartbeatRequest
>();
nodeHeartBeatRequest.SetNodeStatus(nodeStatus);
nodeStatus.SetResponseId(0);
NodeHeartbeatResponse response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest
);
NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 1);
nodeStatus.SetResponseId(response.GetResponseId());
response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 2);
/* try calling with less response id */
response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
NUnit.Framework.Assert.IsTrue(response.GetResponseId() == 2);
nodeStatus.SetResponseId(0);
response = resourceTrackerService.NodeHeartbeat(nodeHeartBeatRequest);
NUnit.Framework.Assert.IsTrue(NodeAction.Resync.Equals(response.GetNodeAction()));
NUnit.Framework.Assert.AreEqual("Too far behind rm response id:2 nm response id:0"
, response.GetDiagnosticsMessage());
}
public virtual void TestDecommissionWithExcludeHosts()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNodesExcludeFilePath, hostFile.GetAbsolutePath());
WriteToHostsFile(string.Empty);
DrainDispatcher dispatcher = new DrainDispatcher();
rm = new _MockRM_162(dispatcher, conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
MockNM nm2 = rm.RegisterNode("host2:5678", 10240);
MockNM nm3 = rm.RegisterNode("localhost:4433", 1024);
dispatcher.Await();
int metricCount = ClusterMetrics.GetMetrics().GetNumDecommisionedNMs();
NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
dispatcher.Await();
// To test that IPs also work
string ip = NetUtils.NormalizeHostName("localhost");
WriteToHostsFile("host2", ip);
rm.GetNodesListManager().RefreshNodes(conf);
CheckDecommissionedNMCount(rm, metricCount + 2);
nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue("The decommisioned metrics are not updated", NodeAction
.Shutdown.Equals(nodeHeartbeat.GetNodeAction()));
nodeHeartbeat = nm3.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue("The decommisioned metrics are not updated", NodeAction
.Shutdown.Equals(nodeHeartbeat.GetNodeAction()));
dispatcher.Await();
WriteToHostsFile(string.Empty);
rm.GetNodesListManager().RefreshNodes(conf);
nm3 = rm.RegisterNode("localhost:4433", 1024);
dispatcher.Await();
nodeHeartbeat = nm3.NodeHeartbeat(true);
dispatcher.Await();
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
// decommissined node is 1 since 1 node is rejoined after updating exclude
// file
CheckDecommissionedNMCount(rm, metricCount + 1);
}
public virtual void TestDecommissionWithIncludeHosts()
{
WriteToHostsFile("localhost", "host1", "host2");
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
MockNM nm2 = rm.RegisterNode("host2:5678", 10240);
MockNM nm3 = rm.RegisterNode("localhost:4433", 1024);
ClusterMetrics metrics = ClusterMetrics.GetMetrics();
System.Diagnostics.Debug.Assert((metrics != null));
int metricCount = metrics.GetNumDecommisionedNMs();
NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
nodeHeartbeat = nm3.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
// To test that IPs also work
string ip = NetUtils.NormalizeHostName("localhost");
WriteToHostsFile("host1", ip);
rm.GetNodesListManager().RefreshNodes(conf);
CheckDecommissionedNMCount(rm, ++metricCount);
nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
NUnit.Framework.Assert.AreEqual(1, ClusterMetrics.GetMetrics().GetNumDecommisionedNMs
());
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue("Node is not decommisioned.", NodeAction.Shutdown.Equals
(nodeHeartbeat.GetNodeAction()));
nodeHeartbeat = nm3.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
NUnit.Framework.Assert.AreEqual(metricCount, ClusterMetrics.GetMetrics().GetNumDecommisionedNMs
());
}
public void Run()
{
int lastHeartBeatID = 0;
while (!this._enclosing.isStopped)
{
try
{
NodeHeartbeatResponse response = null;
NodeStatus nodeStatus = this._enclosing.GetNodeStatus(lastHeartBeatID);
NodeHeartbeatRequest request = NodeHeartbeatRequest.NewInstance(nodeStatus, this.
_enclosing.context.GetContainerTokenSecretManager().GetCurrentKey(), this._enclosing
.context.GetNMTokenSecretManager().GetCurrentKey());
response = this._enclosing.resourceTracker.NodeHeartbeat(request);
this._enclosing.nextHeartBeatInterval = response.GetNextHeartBeatInterval();
this.UpdateMasterKeys(response);
if (response.GetNodeAction() == NodeAction.Shutdown)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeStatusUpdaterImpl.Log.Warn("Recieved SHUTDOWN signal from Resourcemanager as part of heartbeat,"
+ " hence shutting down.");
Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeStatusUpdaterImpl.Log.Warn("Message from ResourceManager: "
+ response.GetDiagnosticsMessage());
this._enclosing.context.SetDecommissioned(true);
this._enclosing.dispatcher.GetEventHandler().Handle(new NodeManagerEvent(NodeManagerEventType
.Shutdown));
break;
}
if (response.GetNodeAction() == NodeAction.Resync)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeStatusUpdaterImpl.Log.Warn("Node is out of sync with ResourceManager,"
+ " hence resyncing.");
Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeStatusUpdaterImpl.Log.Warn("Message from ResourceManager: "
+ response.GetDiagnosticsMessage());
this._enclosing.rmIdentifier = ResourceManagerConstants.RmInvalidIdentifier;
this._enclosing.dispatcher.GetEventHandler().Handle(new NodeManagerEvent(NodeManagerEventType
.Resync));
this._enclosing.pendingCompletedContainers.Clear();
break;
}
this._enclosing.RemoveOrTrackCompletedContainersFromContext(response.GetContainersToBeRemovedFromNM
());
lastHeartBeatID = response.GetResponseId();
IList <ContainerId> containersToCleanup = response.GetContainersToCleanup();
if (!containersToCleanup.IsEmpty())
{
this._enclosing.dispatcher.GetEventHandler().Handle(new CMgrCompletedContainersEvent
(containersToCleanup, CMgrCompletedContainersEvent.Reason.ByResourcemanager));
}
IList <ApplicationId> appsToCleanup = response.GetApplicationsToCleanup();
this._enclosing.TrackAppsForKeepAlive(appsToCleanup);
if (!appsToCleanup.IsEmpty())
{
this._enclosing.dispatcher.GetEventHandler().Handle(new CMgrCompletedAppsEvent(appsToCleanup
, CMgrCompletedAppsEvent.Reason.ByResourcemanager));
}
IDictionary <ApplicationId, ByteBuffer> systemCredentials = response.GetSystemCredentialsForApps
();
if (systemCredentials != null && !systemCredentials.IsEmpty())
{
((NodeManager.NMContext) this._enclosing.context).SetSystemCrendentialsForApps(Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeStatusUpdaterImpl
.ParseCredentials(systemCredentials));
}
}
catch (ConnectException e)
{
this._enclosing.dispatcher.GetEventHandler().Handle(new NodeManagerEvent(NodeManagerEventType
.Shutdown));
throw new YarnRuntimeException(e);
}
catch (Exception e)
{
Org.Apache.Hadoop.Yarn.Server.Nodemanager.NodeStatusUpdaterImpl.Log.Error("Caught exception in status-updater"
, e);
}
finally
{
lock (this._enclosing.heartbeatMonitor)
{
this._enclosing.nextHeartBeatInterval = this._enclosing.nextHeartBeatInterval <=
0 ? YarnConfiguration.DefaultRmNmHeartbeatIntervalMs : this._enclosing.nextHeartBeatInterval;
try
{
Sharpen.Runtime.Wait(this._enclosing.heartbeatMonitor, this._enclosing.nextHeartBeatInterval
);
}
catch (Exception)
{
}
}
}
}
}
public virtual void TestReconnectNode()
{
DrainDispatcher dispatcher = new DrainDispatcher();
rm = new _MockRM_567(this, dispatcher);
rm.Start();
MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
MockNM nm2 = rm.RegisterNode("host2:5678", 5120);
nm1.NodeHeartbeat(true);
nm2.NodeHeartbeat(false);
dispatcher.Await();
CheckUnealthyNMCount(rm, nm2, true, 1);
int expectedNMs = ClusterMetrics.GetMetrics().GetNumActiveNMs();
QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
// TODO Metrics incorrect in case of the FifoScheduler
NUnit.Framework.Assert.AreEqual(5120, metrics.GetAvailableMB());
// reconnect of healthy node
nm1 = rm.RegisterNode("host1:1234", 5120);
NodeHeartbeatResponse response = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(response.GetNodeAction()));
dispatcher.Await();
NUnit.Framework.Assert.AreEqual(expectedNMs, ClusterMetrics.GetMetrics().GetNumActiveNMs
());
CheckUnealthyNMCount(rm, nm2, true, 1);
// reconnect of unhealthy node
nm2 = rm.RegisterNode("host2:5678", 5120);
response = nm2.NodeHeartbeat(false);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(response.GetNodeAction()));
dispatcher.Await();
NUnit.Framework.Assert.AreEqual(expectedNMs, ClusterMetrics.GetMetrics().GetNumActiveNMs
());
CheckUnealthyNMCount(rm, nm2, true, 1);
// unhealthy node changed back to healthy
nm2 = rm.RegisterNode("host2:5678", 5120);
dispatcher.Await();
response = nm2.NodeHeartbeat(true);
response = nm2.NodeHeartbeat(true);
dispatcher.Await();
NUnit.Framework.Assert.AreEqual(5120 + 5120, metrics.GetAvailableMB());
// reconnect of node with changed capability
nm1 = rm.RegisterNode("host2:5678", 10240);
dispatcher.Await();
response = nm1.NodeHeartbeat(true);
dispatcher.Await();
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(response.GetNodeAction()));
NUnit.Framework.Assert.AreEqual(5120 + 10240, metrics.GetAvailableMB());
// reconnect of node with changed capability and running applications
IList <ApplicationId> runningApps = new AList <ApplicationId>();
runningApps.AddItem(ApplicationId.NewInstance(1, 0));
nm1 = rm.RegisterNode("host2:5678", 15360, 2, runningApps);
dispatcher.Await();
response = nm1.NodeHeartbeat(true);
dispatcher.Await();
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(response.GetNodeAction()));
NUnit.Framework.Assert.AreEqual(5120 + 15360, metrics.GetAvailableMB());
// reconnect healthy node changing http port
nm1 = new MockNM("host1:1234", 5120, rm.GetResourceTrackerService());
nm1.SetHttpPort(3);
nm1.RegisterNode();
dispatcher.Await();
response = nm1.NodeHeartbeat(true);
response = nm1.NodeHeartbeat(true);
dispatcher.Await();
RMNode rmNode = rm.GetRMContext().GetRMNodes()[nm1.GetNodeId()];
NUnit.Framework.Assert.AreEqual(3, rmNode.GetHttpPort());
NUnit.Framework.Assert.AreEqual(5120, rmNode.GetTotalCapability().GetMemory());
NUnit.Framework.Assert.AreEqual(5120 + 15360, metrics.GetAvailableMB());
}
// Test does major 6 steps verification.
// Step-1 : AMRMClient send allocate request for 2 container requests
// Step-2 : 2 containers are allocated by RM.
// Step-3 : AM Send 1 containerRequest(cRequest3) and 1 releaseRequests to
// RM
// Step-4 : On RM restart, AM(does not know RM is restarted) sends additional
// containerRequest(cRequest4) and blacklisted nodes.
// Intern RM send resync command
// Step-5 : Allocater after resync command & new containerRequest(cRequest5)
// Step-6 : RM allocates containers i.e cRequest3,cRequest4 and cRequest5
/// <exception cref="System.Exception"/>
public virtual void TestAMRMClientResendsRequestsOnRMRestart()
{
UserGroupInformation.SetLoginUser(null);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// Phase-1 Start 1st RM
TestAMRMClientOnRMRestart.MyResourceManager rm1 = new TestAMRMClientOnRMRestart.MyResourceManager
(conf, memStore);
rm1.Start();
DrainDispatcher dispatcher = (DrainDispatcher)rm1.GetRMContext().GetDispatcher();
// Submit the application
RMApp app = rm1.SubmitApp(1024);
dispatcher.Await();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
nm1.NodeHeartbeat(true);
// Node heartbeat
dispatcher.Await();
ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId();
rm1.SendAMLaunched(appAttemptId);
dispatcher.Await();
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm1.GetRMContext
().GetRMApps()[appAttemptId.GetApplicationId()].GetRMAppAttempt(appAttemptId).GetAMRMToken
();
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ugi.AddTokenIdentifier(token.DecodeIdentifier());
// Step-1 : AMRMClient send allocate request for 2 ContainerRequest
// cRequest1 = h1 and cRequest2 = h1,h2
// blacklisted nodes = h2
AMRMClient <AMRMClient.ContainerRequest> amClient = new TestAMRMClientOnRMRestart.MyAMRMClientImpl
(rm1);
amClient.Init(conf);
amClient.Start();
amClient.RegisterApplicationMaster("Host", 10000, string.Empty);
AMRMClient.ContainerRequest cRequest1 = CreateReq(1, 1024, new string[] { "h1" });
amClient.AddContainerRequest(cRequest1);
AMRMClient.ContainerRequest cRequest2 = CreateReq(1, 1024, new string[] { "h1", "h2" });
amClient.AddContainerRequest(cRequest2);
IList <string> blacklistAdditions = new AList <string>();
IList <string> blacklistRemoval = new AList <string>();
blacklistAdditions.AddItem("h2");
blacklistRemoval.AddItem("h10");
amClient.UpdateBlacklist(blacklistAdditions, blacklistRemoval);
blacklistAdditions.Remove("h2");
// remove from local list
AllocateResponse allocateResponse = amClient.Allocate(0.1f);
dispatcher.Await();
NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse
.GetAllocatedContainers().Count);
// Why 4 ask, why not 3 ask even h2 is blacklisted?
// On blacklisting host,applicationmaster has to remove ask request from
// remoterequest table.Here,test does not remove explicitely
AssertAsksAndReleases(4, 0, rm1);
AssertBlacklistAdditionsAndRemovals(1, 1, rm1);
// Step-2 : NM heart beat is sent.
// On 2nd AM allocate request, RM allocates 2 containers to AM
nm1.NodeHeartbeat(true);
// Node heartbeat
dispatcher.Await();
allocateResponse = amClient.Allocate(0.2f);
dispatcher.Await();
// 2 containers are allocated i.e for cRequest1 and cRequest2.
NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 2, allocateResponse
.GetAllocatedContainers().Count);
AssertAsksAndReleases(0, 0, rm1);
AssertBlacklistAdditionsAndRemovals(0, 0, rm1);
IList <Container> allocatedContainers = allocateResponse.GetAllocatedContainers();
// removed allocated container requests
amClient.RemoveContainerRequest(cRequest1);
amClient.RemoveContainerRequest(cRequest2);
allocateResponse = amClient.Allocate(0.2f);
dispatcher.Await();
NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse
.GetAllocatedContainers().Count);
AssertAsksAndReleases(4, 0, rm1);
AssertBlacklistAdditionsAndRemovals(0, 0, rm1);
// Step-3 : Send 1 containerRequest and 1 releaseRequests to RM
AMRMClient.ContainerRequest cRequest3 = CreateReq(1, 1024, new string[] { "h1" });
amClient.AddContainerRequest(cRequest3);
int pendingRelease = 0;
IEnumerator <Container> it = allocatedContainers.GetEnumerator();
while (it.HasNext())
{
amClient.ReleaseAssignedContainer(it.Next().GetId());
pendingRelease++;
it.Remove();
break;
}
// remove one container
allocateResponse = amClient.Allocate(0.3f);
dispatcher.Await();
NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse
.GetAllocatedContainers().Count);
AssertAsksAndReleases(3, pendingRelease, rm1);
AssertBlacklistAdditionsAndRemovals(0, 0, rm1);
int completedContainer = allocateResponse.GetCompletedContainersStatuses().Count;
pendingRelease -= completedContainer;
// Phase-2 start 2nd RM is up
TestAMRMClientOnRMRestart.MyResourceManager rm2 = new TestAMRMClientOnRMRestart.MyResourceManager
(conf, memStore);
rm2.Start();
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
((TestAMRMClientOnRMRestart.MyAMRMClientImpl)amClient).UpdateRMProxy(rm2);
dispatcher = (DrainDispatcher)rm2.GetRMContext().GetDispatcher();
// NM should be rebooted on heartbeat, even first heartbeat for nm2
NodeHeartbeatResponse hbResponse = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual(NodeAction.Resync, hbResponse.GetNodeAction());
// new NM to represent NM re-register
nm1 = new MockNM("h1:1234", 10240, rm2.GetResourceTrackerService());
nm1.RegisterNode();
nm1.NodeHeartbeat(true);
dispatcher.Await();
blacklistAdditions.AddItem("h3");
amClient.UpdateBlacklist(blacklistAdditions, null);
blacklistAdditions.Remove("h3");
it = allocatedContainers.GetEnumerator();
while (it.HasNext())
{
amClient.ReleaseAssignedContainer(it.Next().GetId());
pendingRelease++;
it.Remove();
}
AMRMClient.ContainerRequest cRequest4 = CreateReq(1, 1024, new string[] { "h1", "h2" });
amClient.AddContainerRequest(cRequest4);
// Step-4 : On RM restart, AM(does not know RM is restarted) sends
// additional
// containerRequest and blacklisted nodes.
// Intern RM send resync command,AMRMClient resend allocate request
allocateResponse = amClient.Allocate(0.3f);
dispatcher.Await();
completedContainer = allocateResponse.GetCompletedContainersStatuses().Count;
pendingRelease -= completedContainer;
AssertAsksAndReleases(4, pendingRelease, rm2);
AssertBlacklistAdditionsAndRemovals(2, 0, rm2);
AMRMClient.ContainerRequest cRequest5 = CreateReq(1, 1024, new string[] { "h1", "h2"
, "h3" });
amClient.AddContainerRequest(cRequest5);
// Step-5 : Allocater after resync command
allocateResponse = amClient.Allocate(0.5f);
dispatcher.Await();
NUnit.Framework.Assert.AreEqual("No of assignments must be 0", 0, allocateResponse
.GetAllocatedContainers().Count);
AssertAsksAndReleases(5, 0, rm2);
AssertBlacklistAdditionsAndRemovals(0, 0, rm2);
int noAssignedContainer = 0;
int count = 5;
while (count-- > 0)
{
nm1.NodeHeartbeat(true);
dispatcher.Await();
allocateResponse = amClient.Allocate(0.5f);
dispatcher.Await();
noAssignedContainer += allocateResponse.GetAllocatedContainers().Count;
if (noAssignedContainer == 3)
{
break;
}
Sharpen.Thread.Sleep(1000);
}
// Step-6 : RM allocates containers i.e cRequest3,cRequest4 and cRequest5
NUnit.Framework.Assert.AreEqual("Number of container should be 3", 3, noAssignedContainer
);
amClient.Stop();
rm1.Stop();
rm2.Stop();
}
// Test verify for
// 1. AM try to unregister without registering
// 2. AM register to RM, and try to unregister immediately after RM restart
/// <exception cref="System.Exception"/>
public virtual void TestAMRMClientForUnregisterAMOnRMRestart()
{
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// Phase-1 Start 1st RM
TestAMRMClientOnRMRestart.MyResourceManager rm1 = new TestAMRMClientOnRMRestart.MyResourceManager
(conf, memStore);
rm1.Start();
DrainDispatcher dispatcher = (DrainDispatcher)rm1.GetRMContext().GetDispatcher();
// Submit the application
RMApp app = rm1.SubmitApp(1024);
dispatcher.Await();
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
nm1.NodeHeartbeat(true);
// Node heartbeat
dispatcher.Await();
ApplicationAttemptId appAttemptId = app.GetCurrentAppAttempt().GetAppAttemptId();
rm1.SendAMLaunched(appAttemptId);
dispatcher.Await();
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> token = rm1.GetRMContext
().GetRMApps()[appAttemptId.GetApplicationId()].GetRMAppAttempt(appAttemptId).GetAMRMToken
();
UserGroupInformation ugi = UserGroupInformation.GetCurrentUser();
ugi.AddTokenIdentifier(token.DecodeIdentifier());
AMRMClient <AMRMClient.ContainerRequest> amClient = new TestAMRMClientOnRMRestart.MyAMRMClientImpl
(rm1);
amClient.Init(conf);
amClient.Start();
amClient.RegisterApplicationMaster("h1", 10000, string.Empty);
amClient.Allocate(0.1f);
// Phase-2 start 2nd RM is up
TestAMRMClientOnRMRestart.MyResourceManager rm2 = new TestAMRMClientOnRMRestart.MyResourceManager
(conf, memStore);
rm2.Start();
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
((TestAMRMClientOnRMRestart.MyAMRMClientImpl)amClient).UpdateRMProxy(rm2);
dispatcher = (DrainDispatcher)rm2.GetRMContext().GetDispatcher();
// NM should be rebooted on heartbeat, even first heartbeat for nm2
NodeHeartbeatResponse hbResponse = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual(NodeAction.Resync, hbResponse.GetNodeAction());
// new NM to represent NM re-register
nm1 = new MockNM("h1:1234", 10240, rm2.GetResourceTrackerService());
ContainerId containerId = ContainerId.NewContainerId(appAttemptId, 1);
NMContainerStatus containerReport = NMContainerStatus.NewInstance(containerId, ContainerState
.Running, Resource.NewInstance(1024, 1), "recover container", 0, Priority.NewInstance
(0), 0);
nm1.RegisterNode(Arrays.AsList(containerReport), null);
nm1.NodeHeartbeat(true);
dispatcher.Await();
amClient.UnregisterApplicationMaster(FinalApplicationStatus.Succeeded, null, null
);
rm2.WaitForState(appAttemptId, RMAppAttemptState.Finishing);
nm1.NodeHeartbeat(appAttemptId, 1, ContainerState.Complete);
rm2.WaitForState(appAttemptId, RMAppAttemptState.Finished);
rm2.WaitForState(app.GetApplicationId(), RMAppState.Finished);
amClient.Stop();
rm1.Stop();
rm2.Stop();
}