/// <exception cref="System.Exception"/>
public virtual void TestAppOnMultiNode()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
conf.Set("yarn.scheduler.capacity.node-locality-delay", "-1");
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
MockNM nm2 = rm.RegisterNode("h2:5678", 10240);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
//request for containers
int request = 13;
am.Allocate("h1", 1000, request, new AList <ContainerId>());
//kick the scheduler
IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
>()).GetAllocatedContainers();
int contReceived = conts.Count;
while (contReceived < 3)
{
//only 3 containers are available on node1
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 3);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(3, conts.Count);
//send node2 heartbeat
conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
contReceived = conts.Count;
while (contReceived < 10)
{
nm2.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 10);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(10, conts.Count);
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestFinishApplicationMasterBeforeRegistering()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(2048);
MockAM am1 = MockRM.LaunchAM(app1, rm, nm1);
FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
.Failed, string.Empty, string.Empty);
try
{
am1.UnregisterAppAttempt(req, false);
NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
);
}
catch (ApplicationMasterNotRegisteredException e)
{
NUnit.Framework.Assert.IsNotNull(e);
NUnit.Framework.Assert.IsNotNull(e.Message);
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Application Master is trying to unregister before registering for:"
));
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
);
}
am1.RegisterAppAttempt();
am1.UnregisterAppAttempt(req, false);
am1.WaitForState(RMAppAttemptState.Finishing);
}
finally
{
if (rm != null)
{
rm.Stop();
}
}
}
public virtual void TestAMLaunchAndCleanup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
TestApplicationMasterLauncher.MyContainerManagerImpl containerManager = new TestApplicationMasterLauncher.MyContainerManagerImpl
();
MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
// kick the scheduling
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.launched == false && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue(containerManager.launched);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId appAttemptId = attempt.GetAppAttemptId();
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), containerManager.attemptIdAtContainerManager
);
NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), containerManager.submitTimeAtContainerManager
);
NUnit.Framework.Assert.AreEqual(app.GetRMAppAttempt(appAttemptId).GetMasterContainer
().GetId().ToString(), containerManager.containerIdAtContainerManager);
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId().ToString(), containerManager.nmHostAtContainerManager
);
NUnit.Framework.Assert.AreEqual(YarnConfiguration.DefaultRmAmMaxAttempts, containerManager
.maxAppAttempts);
MockAM am = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), appAttemptId
);
am.RegisterAppAttempt();
am.UnregisterAppAttempt();
//complete the AM container to finish the app normally
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
waitCount = 0;
while (containerManager.cleanedup == false && waitCount++ < 20)
{
Log.Info("Waiting for AM Cleanup to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue(containerManager.cleanedup);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
// Disable webapp
/// <exception cref="System.Exception"/>
public static void FinishAMAndVerifyAppState(RMApp rmApp, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
rm, MockNM nm, MockAM am)
{
FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
.Succeeded, string.Empty, string.Empty);
am.UnregisterAppAttempt(req, true);
am.WaitForState(RMAppAttemptState.Finishing);
nm.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.WaitForState(rmApp.GetApplicationId(), RMAppState.Finished);
}
/// <exception cref="System.Exception"/>
public virtual void TestAppWithNoContainers()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
// Test Kill an app while the app is finishing in the meanwhile.
/// <exception cref="System.Exception"/>
public virtual void TestKillFinishingApp()
{
// this dispatcher ignores RMAppAttemptEventType.KILL event
Dispatcher dispatcher = new _AsyncDispatcher_654();
MockRM rm1 = new _MockRM_677(dispatcher, conf);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
rm1.KillApp(app1.GetApplicationId());
FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
.Succeeded, string.Empty, string.Empty);
am1.UnregisterAppAttempt(req, true);
rm1.WaitForState(am1.GetApplicationAttemptId(), RMAppAttemptState.Finishing);
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
rm1.WaitForState(am1.GetApplicationAttemptId(), RMAppAttemptState.Finished);
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Finished);
}
public virtual void TestAppCleanup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM();
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5000);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
//request for containers
int request = 2;
am.Allocate("127.0.0.1", 1000, request, new AList <ContainerId>());
//kick the scheduler
nm1.NodeHeartbeat(true);
IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
>()).GetAllocatedContainers();
int contReceived = conts.Count;
int waitCount = 0;
while (contReceived < request && waitCount++ < 200)
{
Log.Info("Got " + contReceived + " containers. Waiting to get " + request);
Sharpen.Thread.Sleep(100);
conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
contReceived += conts.Count;
nm1.NodeHeartbeat(true);
}
NUnit.Framework.Assert.AreEqual(request, contReceived);
am.UnregisterAppAttempt();
NodeHeartbeatResponse resp = nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState
.Complete);
am.WaitForState(RMAppAttemptState.Finished);
//currently only containers are cleaned via this
//AM container is cleaned via container launcher
resp = nm1.NodeHeartbeat(true);
IList <ContainerId> containersToCleanup = resp.GetContainersToCleanup();
IList <ApplicationId> appsToCleanup = resp.GetApplicationsToCleanup();
int numCleanedContainers = containersToCleanup.Count;
int numCleanedApps = appsToCleanup.Count;
waitCount = 0;
while ((numCleanedContainers < 2 || numCleanedApps < 1) && waitCount++ < 200)
{
Log.Info("Waiting to get cleanup events.. cleanedConts: " + numCleanedContainers
+ " cleanedApps: " + numCleanedApps);
Sharpen.Thread.Sleep(100);
resp = nm1.NodeHeartbeat(true);
IList <ContainerId> deltaContainersToCleanup = resp.GetContainersToCleanup();
IList <ApplicationId> deltaAppsToCleanup = resp.GetApplicationsToCleanup();
// Add the deltas to the global list
Sharpen.Collections.AddAll(containersToCleanup, deltaContainersToCleanup);
Sharpen.Collections.AddAll(appsToCleanup, deltaAppsToCleanup);
// Update counts now
numCleanedContainers = containersToCleanup.Count;
numCleanedApps = appsToCleanup.Count;
}
NUnit.Framework.Assert.AreEqual(1, appsToCleanup.Count);
NUnit.Framework.Assert.AreEqual(app.GetApplicationId(), appsToCleanup[0]);
NUnit.Framework.Assert.AreEqual(1, numCleanedApps);
NUnit.Framework.Assert.AreEqual(2, numCleanedContainers);
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestallocateBeforeAMRegistration()
{
Logger rootLogger = LogManager.GetRootLogger();
bool thrown = false;
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM();
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5000);
RMApp app = rm.SubmitApp(2000);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
// request for containers
int request = 2;
AllocateResponse ar = null;
try
{
ar = am.Allocate("h1", 1000, request, new AList <ContainerId>());
NUnit.Framework.Assert.Fail();
}
catch (ApplicationMasterNotRegisteredException)
{
}
// kick the scheduler
nm1.NodeHeartbeat(true);
AllocateResponse amrs = null;
try
{
amrs = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>());
NUnit.Framework.Assert.Fail();
}
catch (ApplicationMasterNotRegisteredException)
{
}
am.RegisterAppAttempt();
try
{
am.RegisterAppAttempt(false);
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.AreEqual("Application Master is already registered : " + attempt
.GetAppAttemptId().GetApplicationId(), e.Message);
}
// Simulate an AM that was disconnected and app attempt was removed
// (responseMap does not contain attemptid)
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
try
{
amrs = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>());
NUnit.Framework.Assert.Fail();
}
catch (ApplicationAttemptNotFoundException)
{
}
}
/// <exception cref="System.Exception"/>
public virtual void TestNMToken()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 10000);
NMTokenSecretManagerInRM nmTokenSecretManager = rm.GetRMContext().GetNMTokenSecretManager
();
// submitting new application
RMApp app = rm.SubmitApp(1000);
// start scheduling.
nm1.NodeHeartbeat(true);
// Starting application attempt and launching
// It should get registered with NMTokenSecretManager.
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptRegistered
(attempt.GetAppAttemptId()));
// This will register application master.
am.RegisterAppAttempt();
AList <Container> containersReceivedForNM1 = new AList <Container>();
IList <ContainerId> releaseContainerList = new AList <ContainerId>();
Dictionary <string, Token> nmTokens = new Dictionary <string, Token>();
// initially requesting 2 containers.
AllocateResponse response = am.Allocate("h1", 1000, 2, releaseContainerList);
NUnit.Framework.Assert.AreEqual(0, response.GetAllocatedContainers().Count);
AllocateContainersAndValidateNMTokens(am, containersReceivedForNM1, 2, nmTokens,
nm1);
NUnit.Framework.Assert.AreEqual(1, nmTokens.Count);
// requesting 2 more containers.
response = am.Allocate("h1", 1000, 2, releaseContainerList);
NUnit.Framework.Assert.AreEqual(0, response.GetAllocatedContainers().Count);
AllocateContainersAndValidateNMTokens(am, containersReceivedForNM1, 4, nmTokens,
nm1);
NUnit.Framework.Assert.AreEqual(1, nmTokens.Count);
// We will be simulating NM restart so restarting newly added h2:1234
// NM 2 now registers.
MockNM nm2 = rm.RegisterNode("h2:1234", 10000);
nm2.NodeHeartbeat(true);
AList <Container> containersReceivedForNM2 = new AList <Container>();
response = am.Allocate("h2", 1000, 2, releaseContainerList);
NUnit.Framework.Assert.AreEqual(0, response.GetAllocatedContainers().Count);
AllocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 2, nmTokens,
nm2);
NUnit.Framework.Assert.AreEqual(2, nmTokens.Count);
// Simulating NM-2 restart.
nm2 = rm.RegisterNode("h2:1234", 10000);
// Wait for reconnect to make it through the RM and create a new RMNode
IDictionary <NodeId, RMNode> nodes = rm.GetRMContext().GetRMNodes();
while (nodes[nm2.GetNodeId()].GetLastNodeHeartBeatResponse().GetResponseId() > 0)
{
Sharpen.Thread.Sleep(WaitSleepMs);
}
int interval = 40;
// Wait for nm Token to be cleared.
while (nmTokenSecretManager.IsApplicationAttemptNMTokenPresent(attempt.GetAppAttemptId
(), nm2.GetNodeId()) && interval-- > 0)
{
Log.Info("waiting for nmToken to be cleared for : " + nm2.GetNodeId());
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptRegistered
(attempt.GetAppAttemptId()));
// removing NMToken for h2:1234
Sharpen.Collections.Remove(nmTokens, nm2.GetNodeId().ToString());
NUnit.Framework.Assert.AreEqual(1, nmTokens.Count);
// We should again receive the NMToken.
response = am.Allocate("h2", 1000, 2, releaseContainerList);
NUnit.Framework.Assert.AreEqual(0, response.GetAllocatedContainers().Count);
AllocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 4, nmTokens,
nm2);
NUnit.Framework.Assert.AreEqual(2, nmTokens.Count);
// Now rolling over NMToken masterKey. it should resend the NMToken in
// next allocate call.
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm2.GetNodeId()));
nmTokenSecretManager.RollMasterKey();
nmTokenSecretManager.ActivateNextMasterKey();
NUnit.Framework.Assert.IsFalse(nmTokenSecretManager.IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
NUnit.Framework.Assert.IsFalse(nmTokenSecretManager.IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm2.GetNodeId()));
// It should not remove application attempt entry.
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptRegistered
(attempt.GetAppAttemptId()));
nmTokens.Clear();
NUnit.Framework.Assert.AreEqual(0, nmTokens.Count);
// We should again receive the NMToken.
response = am.Allocate("h2", 1000, 1, releaseContainerList);
NUnit.Framework.Assert.AreEqual(0, response.GetAllocatedContainers().Count);
AllocateContainersAndValidateNMTokens(am, containersReceivedForNM2, 5, nmTokens,
nm2);
NUnit.Framework.Assert.AreEqual(1, nmTokens.Count);
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm2.GetNodeId()));
// After AM is finished making sure that nmtoken entry for app
NUnit.Framework.Assert.IsTrue(nmTokenSecretManager.IsApplicationAttemptRegistered
(attempt.GetAppAttemptId()));
am.UnregisterAppAttempt();
// marking all the containers as finished.
foreach (Container container in containersReceivedForNM1)
{
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), container.GetId().GetContainerId(),
ContainerState.Complete);
}
foreach (Container container_1 in containersReceivedForNM2)
{
nm2.NodeHeartbeat(attempt.GetAppAttemptId(), container_1.GetId().GetContainerId()
, ContainerState.Complete);
}
nm1.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
NUnit.Framework.Assert.IsFalse(nmTokenSecretManager.IsApplicationAttemptRegistered
(attempt.GetAppAttemptId()));
}
finally
{
rm.Stop();
}
}
