public virtual void TestAMLaunchAndCleanup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
TestApplicationMasterLauncher.MyContainerManagerImpl containerManager = new TestApplicationMasterLauncher.MyContainerManagerImpl
();
MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
// kick the scheduling
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.launched == false && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue(containerManager.launched);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId appAttemptId = attempt.GetAppAttemptId();
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), containerManager.attemptIdAtContainerManager
);
NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), containerManager.submitTimeAtContainerManager
);
NUnit.Framework.Assert.AreEqual(app.GetRMAppAttempt(appAttemptId).GetMasterContainer
().GetId().ToString(), containerManager.containerIdAtContainerManager);
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId().ToString(), containerManager.nmHostAtContainerManager
);
NUnit.Framework.Assert.AreEqual(YarnConfiguration.DefaultRmAmMaxAttempts, containerManager
.maxAppAttempts);
MockAM am = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), appAttemptId
);
am.RegisterAppAttempt();
am.UnregisterAppAttempt();
//complete the AM container to finish the app normally
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
waitCount = 0;
while (containerManager.cleanedup == false && waitCount++ < 20)
{
Log.Info("Waiting for AM Cleanup to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue(containerManager.cleanedup);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
// Disable webapp
/// <exception cref="System.Exception"/>
public static void FinishAMAndVerifyAppState(RMApp rmApp, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
rm, MockNM nm, MockAM am)
{
FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
.Succeeded, string.Empty, string.Empty);
am.UnregisterAppAttempt(req, true);
am.WaitForState(RMAppAttemptState.Finishing);
nm.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.WaitForState(rmApp.GetApplicationId(), RMAppState.Finished);
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidatedAMHostPortOnAMRestart()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// a failed app
RMApp app2 = rm1.SubmitApp(200);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app2.GetApplicationId(), RMAppState.Accepted);
// before new attempt is launched, the app report returns the invalid AM
// host and port.
GetApplicationReportRequest request1 = GetApplicationReportRequest.NewInstance(app2
.GetApplicationId());
ApplicationReport report1 = rm1.GetClientRMService().GetApplicationReport(request1
).GetApplicationReport();
NUnit.Framework.Assert.AreEqual("N/A", report1.GetHost());
NUnit.Framework.Assert.AreEqual(-1, report1.GetRpcPort());
}
/// <exception cref="System.Exception"/>
public virtual void SendAMLaunchFailed(ApplicationAttemptId appAttemptId)
{
MockAM am = new MockAM(GetRMContext(), masterService, appAttemptId);
am.WaitForState(RMAppAttemptState.Allocated);
GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptEvent(appAttemptId
, RMAppAttemptEventType.LaunchFailed, "Failed"));
}
/// <exception cref="System.Exception"/>
public virtual void TestAppOnMultiNode()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
conf.Set("yarn.scheduler.capacity.node-locality-delay", "-1");
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
MockNM nm2 = rm.RegisterNode("h2:5678", 10240);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
//request for containers
int request = 13;
am.Allocate("h1", 1000, request, new AList <ContainerId>());
//kick the scheduler
IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
>()).GetAllocatedContainers();
int contReceived = conts.Count;
while (contReceived < 3)
{
//only 3 containers are available on node1
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 3);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(3, conts.Count);
//send node2 heartbeat
conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
contReceived = conts.Count;
while (contReceived < 10)
{
nm2.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 10);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(10, conts.Count);
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
// This is to test AM Host and rpc port are invalidated after the am attempt
// is killed or failed, so that client doesn't get the wrong information.
/// <exception cref="System.Exception"/>
public virtual void TestInvalidateAMHostPortWhenAMFailedOrKilled()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MockRM rm1 = new MockRM(conf);
rm1.Start();
// a succeeded app
RMApp app1 = rm1.SubmitApp(200);
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm1, nm1, am1);
// a failed app
RMApp app2 = rm1.SubmitApp(200);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app2.GetApplicationId(), RMAppState.Failed);
// a killed app
RMApp app3 = rm1.SubmitApp(200);
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm1);
rm1.KillApp(app3.GetApplicationId());
rm1.WaitForState(app3.GetApplicationId(), RMAppState.Killed);
rm1.WaitForState(am3.GetApplicationAttemptId(), RMAppAttemptState.Killed);
GetApplicationsRequest request1 = GetApplicationsRequest.NewInstance(EnumSet.Of(YarnApplicationState
.Finished, YarnApplicationState.Killed, YarnApplicationState.Failed));
GetApplicationsResponse response1 = rm1.GetClientRMService().GetApplications(request1
);
IList <ApplicationReport> appList1 = response1.GetApplicationList();
NUnit.Framework.Assert.AreEqual(3, appList1.Count);
foreach (ApplicationReport report in appList1)
{
// killed/failed apps host and rpc port are invalidated.
if (report.GetApplicationId().Equals(app2.GetApplicationId()) || report.GetApplicationId
().Equals(app3.GetApplicationId()))
{
NUnit.Framework.Assert.AreEqual("N/A", report.GetHost());
NUnit.Framework.Assert.AreEqual(-1, report.GetRpcPort());
}
// succeeded app's host and rpc port is not invalidated
if (report.GetApplicationId().Equals(app1.GetApplicationId()))
{
NUnit.Framework.Assert.IsFalse(report.GetHost().Equals("N/A"));
NUnit.Framework.Assert.IsTrue(report.GetRpcPort() != -1);
}
}
}
// from AMLauncher
/// <exception cref="System.Exception"/>
public virtual MockAM SendAMLaunched(ApplicationAttemptId appAttemptId)
{
MockAM am = new MockAM(GetRMContext(), masterService, appAttemptId);
am.WaitForState(RMAppAttemptState.Allocated);
//create and set AMRMToken
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = this.rmContext
.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId);
((RMAppAttemptImpl)this.rmContext.GetRMApps()[appAttemptId.GetApplicationId()].GetRMAppAttempt
(appAttemptId)).SetAMRMToken(amrmToken);
GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptEvent(appAttemptId
, RMAppAttemptEventType.Launched));
return(am);
}
/// <exception cref="System.Exception"/>
public virtual void TestFinishApplicationMasterBeforeRegistering()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(2048);
MockAM am1 = MockRM.LaunchAM(app1, rm, nm1);
FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
.Failed, string.Empty, string.Empty);
try
{
am1.UnregisterAppAttempt(req, false);
NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
);
}
catch (ApplicationMasterNotRegisteredException e)
{
NUnit.Framework.Assert.IsNotNull(e);
NUnit.Framework.Assert.IsNotNull(e.Message);
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Application Master is trying to unregister before registering for:"
));
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
);
}
am1.RegisterAppAttempt();
am1.UnregisterAppAttempt(req, false);
am1.WaitForState(RMAppAttemptState.Finishing);
}
finally
{
if (rm != null)
{
rm.Stop();
}
}
}
/// <summary>Validate killing an application when it is at accepted state.</summary>
/// <exception cref="System.Exception">exception</exception>
public virtual void TestApplicationKillAtAcceptedState()
{
Dispatcher dispatcher = new _AsyncDispatcher_573();
MockRM rm = new _MockRM_596(dispatcher, conf);
// test metrics
QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
int appsKilled = metrics.GetAppsKilled();
int appsSubmitted = metrics.GetAppsSubmitted();
rm.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());
nm1.RegisterNode();
// a failed app
RMApp application = rm.SubmitApp(200);
MockAM am = MockRM.LaunchAM(application, rm, nm1);
am.WaitForState(RMAppAttemptState.Launched);
nm1.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Running);
rm.WaitForState(application.GetApplicationId(), RMAppState.Accepted);
// Now kill the application before new attempt is launched, the app report
// returns the invalid AM host and port.
KillApplicationRequest request = KillApplicationRequest.NewInstance(application.GetApplicationId
());
rm.GetClientRMService().ForceKillApplication(request);
// Specific test for YARN-1689 follows
// Now let's say a race causes AM to register now. This should not crash RM.
am.RegisterAppAttempt(false);
// We explicitly intercepted the kill-event to RMAppAttempt, so app should
// still be in KILLING state.
rm.WaitForState(application.GetApplicationId(), RMAppState.Killing);
// AM should now be in running
rm.WaitForState(am.GetApplicationAttemptId(), RMAppAttemptState.Running);
// Simulate that appAttempt is killed.
rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppEvent(application
.GetApplicationId(), RMAppEventType.AttemptKilled));
rm.WaitForState(application.GetApplicationId(), RMAppState.Killed);
// test metrics
metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
NUnit.Framework.Assert.AreEqual(appsKilled + 1, metrics.GetAppsKilled());
NUnit.Framework.Assert.AreEqual(appsSubmitted + 1, metrics.GetAppsSubmitted());
}
/// <exception cref="System.Exception"/>
public virtual void TestAppWithNoContainers()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
/// <exception cref="System.Exception"/>
private void AmRestartTests(bool keepRunningContainers)
{
MockRM rm = new MockRM(conf);
rm.Start();
RMApp app = rm.SubmitApp(200, "name", "user", new Dictionary <ApplicationAccessType
, string>(), false, "default", -1, null, "MAPREDUCE", false, keepRunningContainers
);
MockNM nm = new MockNM("127.0.0.1:1234", 10240, rm.GetResourceTrackerService());
nm.RegisterNode();
MockAM am0 = MockRM.LaunchAndRegisterAM(app, rm, nm);
int NumContainers = 1;
// allocate NUM_CONTAINERS containers
am0.Allocate("127.0.0.1", 1024, NumContainers, new AList <ContainerId>());
nm.NodeHeartbeat(true);
// wait for containers to be allocated.
IList <Container> containers = am0.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
while (containers.Count != NumContainers)
{
nm.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, am0.Allocate(new AList <ResourceRequest>(),
new AList <ContainerId>()).GetAllocatedContainers());
Sharpen.Thread.Sleep(200);
}
// launch the 2nd container.
ContainerId containerId2 = ContainerId.NewContainerId(am0.GetApplicationAttemptId
(), 2);
nm.NodeHeartbeat(am0.GetApplicationAttemptId(), containerId2.GetContainerId(), ContainerState
.Running);
rm.WaitForState(nm, containerId2, RMContainerState.Running);
// Capture the containers here so the metrics can be calculated after the
// app has completed.
ICollection <RMContainer> rmContainers = rm.scheduler.GetSchedulerAppInfo(am0.GetApplicationAttemptId
()).GetLiveContainers();
// fail the first app attempt by sending CONTAINER_FINISHED event without
// registering.
ContainerId amContainerId = app.GetCurrentAppAttempt().GetMasterContainer().GetId
();
nm.NodeHeartbeat(am0.GetApplicationAttemptId(), amContainerId.GetContainerId(), ContainerState
.Complete);
am0.WaitForState(RMAppAttemptState.Failed);
long memorySeconds = 0;
long vcoreSeconds = 0;
// Calculate container usage metrics for first attempt.
if (keepRunningContainers)
{
// Only calculate the usage for the one container that has completed.
foreach (RMContainer c in rmContainers)
{
if (c.GetContainerId().Equals(amContainerId))
{
AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(c);
memorySeconds += ru.GetMemorySeconds();
vcoreSeconds += ru.GetVcoreSeconds();
}
else
{
// The remaining container should be RUNNING.
NUnit.Framework.Assert.IsTrue("After first attempt failed, remaining container "
+ "should still be running. ", c.GetContainerState().Equals(ContainerState.Running
));
}
}
}
else
{
// If keepRunningContainers is false, all live containers should now
// be completed. Calculate the resource usage metrics for all of them.
foreach (RMContainer c in rmContainers)
{
AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(c);
memorySeconds += ru.GetMemorySeconds();
vcoreSeconds += ru.GetVcoreSeconds();
}
}
// wait for app to start a new attempt.
rm.WaitForState(app.GetApplicationId(), RMAppState.Accepted);
// assert this is a new AM.
RMAppAttempt attempt2 = app.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsFalse(attempt2.GetAppAttemptId().Equals(am0.GetApplicationAttemptId
()));
// launch the new AM
nm.NodeHeartbeat(true);
MockAM am1 = rm.SendAMLaunched(attempt2.GetAppAttemptId());
am1.RegisterAppAttempt();
// allocate NUM_CONTAINERS containers
am1.Allocate("127.0.0.1", 1024, NumContainers, new AList <ContainerId>());
nm.NodeHeartbeat(true);
// wait for containers to be allocated.
containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>())
.GetAllocatedContainers();
while (containers.Count != NumContainers)
{
nm.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, am1.Allocate(new AList <ResourceRequest>(),
new AList <ContainerId>()).GetAllocatedContainers());
Sharpen.Thread.Sleep(200);
}
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
// Capture running containers for later use by metrics calculations.
rmContainers = rm.scheduler.GetSchedulerAppInfo(attempt2.GetAppAttemptId()).GetLiveContainers
();
// complete container by sending the container complete event which has
// earlier attempt's attemptId
amContainerId = app.GetCurrentAppAttempt().GetMasterContainer().GetId();
nm.NodeHeartbeat(am0.GetApplicationAttemptId(), amContainerId.GetContainerId(), ContainerState
.Complete);
MockRM.FinishAMAndVerifyAppState(app, rm, nm, am1);
// Calculate container usage metrics for second attempt.
foreach (RMContainer c_1 in rmContainers)
{
AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(c_1);
memorySeconds += ru.GetMemorySeconds();
vcoreSeconds += ru.GetVcoreSeconds();
}
RMAppMetrics rmAppMetrics = app.GetRMAppMetrics();
NUnit.Framework.Assert.AreEqual("Unexcpected MemorySeconds value", memorySeconds,
rmAppMetrics.GetMemorySeconds());
NUnit.Framework.Assert.AreEqual("Unexpected VcoreSeconds value", vcoreSeconds, rmAppMetrics
.GetVcoreSeconds());
rm.Stop();
return;
}
public virtual void TestAppCleanup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM();
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5000);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
//request for containers
int request = 2;
am.Allocate("127.0.0.1", 1000, request, new AList <ContainerId>());
//kick the scheduler
nm1.NodeHeartbeat(true);
IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
>()).GetAllocatedContainers();
int contReceived = conts.Count;
int waitCount = 0;
while (contReceived < request && waitCount++ < 200)
{
Log.Info("Got " + contReceived + " containers. Waiting to get " + request);
Sharpen.Thread.Sleep(100);
conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
contReceived += conts.Count;
nm1.NodeHeartbeat(true);
}
NUnit.Framework.Assert.AreEqual(request, contReceived);
am.UnregisterAppAttempt();
NodeHeartbeatResponse resp = nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState
.Complete);
am.WaitForState(RMAppAttemptState.Finished);
//currently only containers are cleaned via this
//AM container is cleaned via container launcher
resp = nm1.NodeHeartbeat(true);
IList <ContainerId> containersToCleanup = resp.GetContainersToCleanup();
IList <ApplicationId> appsToCleanup = resp.GetApplicationsToCleanup();
int numCleanedContainers = containersToCleanup.Count;
int numCleanedApps = appsToCleanup.Count;
waitCount = 0;
while ((numCleanedContainers < 2 || numCleanedApps < 1) && waitCount++ < 200)
{
Log.Info("Waiting to get cleanup events.. cleanedConts: " + numCleanedContainers
+ " cleanedApps: " + numCleanedApps);
Sharpen.Thread.Sleep(100);
resp = nm1.NodeHeartbeat(true);
IList <ContainerId> deltaContainersToCleanup = resp.GetContainersToCleanup();
IList <ApplicationId> deltaAppsToCleanup = resp.GetApplicationsToCleanup();
// Add the deltas to the global list
Sharpen.Collections.AddAll(containersToCleanup, deltaContainersToCleanup);
Sharpen.Collections.AddAll(appsToCleanup, deltaAppsToCleanup);
// Update counts now
numCleanedContainers = containersToCleanup.Count;
numCleanedApps = appsToCleanup.Count;
}
NUnit.Framework.Assert.AreEqual(1, appsToCleanup.Count);
NUnit.Framework.Assert.AreEqual(app.GetApplicationId(), appsToCleanup[0]);
NUnit.Framework.Assert.AreEqual(1, numCleanedApps);
NUnit.Framework.Assert.AreEqual(2, numCleanedContainers);
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestallocateBeforeAMRegistration()
{
Logger rootLogger = LogManager.GetRootLogger();
bool thrown = false;
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM();
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5000);
RMApp app = rm.SubmitApp(2000);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
// request for containers
int request = 2;
AllocateResponse ar = null;
try
{
ar = am.Allocate("h1", 1000, request, new AList <ContainerId>());
NUnit.Framework.Assert.Fail();
}
catch (ApplicationMasterNotRegisteredException)
{
}
// kick the scheduler
nm1.NodeHeartbeat(true);
AllocateResponse amrs = null;
try
{
amrs = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>());
NUnit.Framework.Assert.Fail();
}
catch (ApplicationMasterNotRegisteredException)
{
}
am.RegisterAppAttempt();
try
{
am.RegisterAppAttempt(false);
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.AreEqual("Application Master is already registered : " + attempt
.GetAppAttemptId().GetApplicationId(), e.Message);
}
// Simulate an AM that was disconnected and app attempt was removed
// (responseMap does not contain attemptid)
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
try
{
amrs = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>());
NUnit.Framework.Assert.Fail();
}
catch (ApplicationAttemptNotFoundException)
{
}
}
/// <exception cref="System.Exception"/>
public virtual void TestNMToken()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 10000);
NMTokenSecretManagerInRM nmTokenSecretManager = rm.GetRMContext().GetNMTokenSecretManager
();
// submitting new application
RMApp app = rm.SubmitApp(1000);
// start scheduling.
nm1.NodeHeartbeat(true);
// Starting application attempt and launching
// It should get registered with NMTokenSecretManager.
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptRegistered
(attempt.GetAppAttemptId()));
// This will register application master.
am.RegisterAppAttempt();
AList <Container> containersReceivedForNM1 = new AList <Container>();
IList <ContainerId> releaseContainerList = new AList <ContainerId>();
Dictionary <string, Token> nmTokens = new Dictionary <string, Token>();
// initially requesting 2 containers.
AllocateResponse response = am.Allocate("h1", 1000, 2, releaseContainerList);
NUnit.Framework.Assert.AreEqual(0, response.GetAllocatedContainers().Count);
AllocateContainersAndValidateNMTokens(am, containersReceivedForNM1, 2, nmTokens,
nm1);
NUnit.Framework.Assert.AreEqual(1, nmTokens.Count);
// requesting 2 more containers.
response = am.Allocate("h1", 1000, 2, releaseContainerList);
NUnit.Framework.Assert.AreEqual(0, response.GetAllocatedContainers().Count);
AllocateContainersAndValidateNMTokens(am, containersReceivedForNM1, 4, nmTokens,
nm1);
NUnit.Framework.Assert.AreEqual(1, nmTokens.Count);
// We will be simulating NM restart so restarting newly added h2:1234
// NM 2 now registers.
MockNM nm2 = rm.RegisterNode("h2:1234", 10000);
nm2.NodeHeartbeat(true);
AList <Container> containersReceivedForNM2 = new AList <Container>();
response = am.Allocate("h2", 1000, 2, releaseContainerList);
NUnit.Framework.Assert.AreEqual(0, response.GetAllocatedContainers().Count);
AllocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 2, nmTokens,
nm2);
NUnit.Framework.Assert.AreEqual(2, nmTokens.Count);
// Simulating NM-2 restart.
nm2 = rm.RegisterNode("h2:1234", 10000);
// Wait for reconnect to make it through the RM and create a new RMNode
IDictionary <NodeId, RMNode> nodes = rm.GetRMContext().GetRMNodes();
while (nodes[nm2.GetNodeId()].GetLastNodeHeartBeatResponse().GetResponseId() > 0)
{
Sharpen.Thread.Sleep(WaitSleepMs);
}
int interval = 40;
// Wait for nm Token to be cleared.
while (nmTokenSecretManager.IsApplicationAttemptNMTokenPresent(attempt.GetAppAttemptId
(), nm2.GetNodeId()) && interval-- > 0)
{
Log.Info("waiting for nmToken to be cleared for : " + nm2.GetNodeId());
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptRegistered
(attempt.GetAppAttemptId()));
// removing NMToken for h2:1234
Sharpen.Collections.Remove(nmTokens, nm2.GetNodeId().ToString());
NUnit.Framework.Assert.AreEqual(1, nmTokens.Count);
// We should again receive the NMToken.
response = am.Allocate("h2", 1000, 2, releaseContainerList);
NUnit.Framework.Assert.AreEqual(0, response.GetAllocatedContainers().Count);
AllocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 4, nmTokens,
nm2);
NUnit.Framework.Assert.AreEqual(2, nmTokens.Count);
// Now rolling over NMToken masterKey. it should resend the NMToken in
// next allocate call.
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm2.GetNodeId()));
nmTokenSecretManager.RollMasterKey();
nmTokenSecretManager.ActivateNextMasterKey();
NUnit.Framework.Assert.IsFalse(nmTokenSecretManager.IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
NUnit.Framework.Assert.IsFalse(nmTokenSecretManager.IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm2.GetNodeId()));
// It should not remove application attempt entry.
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptRegistered
(attempt.GetAppAttemptId()));
nmTokens.Clear();
NUnit.Framework.Assert.AreEqual(0, nmTokens.Count);
// We should again receive the NMToken.
response = am.Allocate("h2", 1000, 1, releaseContainerList);
NUnit.Framework.Assert.AreEqual(0, response.GetAllocatedContainers().Count);
AllocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 5, nmTokens,
nm2);
NUnit.Framework.Assert.AreEqual(1, nmTokens.Count);
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm2.GetNodeId()));
// After AM is finished making sure that nmtoken entry for app
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptRegistered
(attempt.GetAppAttemptId()));
am.UnregisterAppAttempt();
// marking all the containers as finished.
foreach (Container container in containersReceivedForNM1)
{
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), container.GetId().GetContainerId(),
ContainerState.Complete);
}
foreach (Container container_1 in containersReceivedForNM2)
{
nm2.NodeHeartbeat(attempt.GetAppAttemptId(), container_1.GetId().GetContainerId()
, ContainerState.Complete);
}
nm1.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
NUnit.Framework.Assert.IsFalse(nmTokenSecretManager.IsApplicationAttemptRegistered
(attempt.GetAppAttemptId()));
}
finally
{
rm.Stop();
}
}
