// Test RM restarts after AM container is preempted, new RM should not count
// AM preemption failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestPreemptedAMRestartOnRMRestart()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler();
ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 1);
// Forcibly preempt the am container;
scheduler.KillContainer(scheduler.GetRMContainer(amContainer));
am1.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry());
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// state store has 1 attempt stored.
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
NUnit.Framework.Assert.AreEqual(1, appState.GetAttemptCount());
// attempt stored has the preempted container exit status.
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Preempted, appState.GetAttempt
(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode();
rm2.Start();
// Restarted RM should re-launch the am.
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt2 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt2.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocateWithLabels()
{
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm1 = new _MockRM_638(this, GetConfigurationWithQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
// label = x
MockNM nm2 = rm1.RegisterNode("h2:1234", 8000);
// label = y
MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
// label = <empty>
ContainerId containerId;
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm3);
// request a container.
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h1");
// launch an app to queue b1 (label = y), and check all container will
// be allocated in h2
RMApp app2 = rm1.SubmitApp(200, "app", "user", null, "b1");
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm3);
// request a container.
am2.Allocate("*", 1024, 1, new AList <ContainerId>(), "y");
containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am2.GetApplicationAttemptId(), containerId, rm1, "h2");
// launch an app to queue c1 (label = ""), and check all container will
// be allocated in h3
RMApp app3 = rm1.SubmitApp(200, "app", "user", null, "c1");
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm3);
// request a container.
am3.Allocate("*", 1024, 1, new AList <ContainerId>());
containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3");
rm1.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocationWithSingleUserLimits()
{
RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
mgr.Init(conf);
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm1 = new _MockRM_471(mgr, GetConfigurationWithDefaultQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
// label = x
rm1.RegisterNode("h2:1234", 8000);
// label = y
MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
// label = <empty>
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// A has only 10% of x, so it can only allocate one container in label=empty
ContainerId containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 2);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty);
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
// Cannot allocate 2nd label=empty container
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 3);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
// A has default user limit = 100, so it can use all resource in label = x
// We can allocate floor(8000 / 1024) = 7 containers
for (int id = 3; id <= 8; id++)
{
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), id);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
}
rm1.Close();
}
/// <exception cref="System.Exception"/>
private LogAggregationContext GetLogAggregationContextFromContainerToken(MockRM rm1
, MockNM nm1, LogAggregationContext logAggregationContext)
{
RMApp app2 = rm1.SubmitApp(200, logAggregationContext);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(true);
// request a container.
am2.Allocate("127.0.0.1", 512, 1, new AList <ContainerId>());
ContainerId containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(
), 2);
rm1.WaitForState(nm1, containerId, RMContainerState.Allocated);
// acquire the container.
IList <Container> containers = am2.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
NUnit.Framework.Assert.AreEqual(containerId, containers[0].GetId());
// container token is generated.
NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken());
ContainerTokenIdentifier token = BuilderUtils.NewContainerTokenIdentifier(containers
[0].GetContainerToken());
return(token.GetLogAggregationContext());
}
public virtual void TestNormalContainerAllocationWhenDNSUnavailable()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
RMApp app1 = rm1.SubmitApp(200);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
// acquire the container.
SecurityUtilTestHelper.SetTokenServiceUseIp(true);
IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
// not able to fetch the container;
NUnit.Framework.Assert.AreEqual(0, containers.Count);
SecurityUtilTestHelper.SetTokenServiceUseIp(false);
containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>())
.GetAllocatedContainers();
// should be able to fetch the container;
NUnit.Framework.Assert.AreEqual(1, containers.Count);
}
public virtual void TestContainerTokenGeneratedOnPullRequest()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("127.0.0.1:1234", 8000);
RMApp app1 = rm1.SubmitApp(200);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
RMContainer container = rm1.GetResourceScheduler().GetRMContainer(containerId2);
// no container token is generated.
NUnit.Framework.Assert.AreEqual(containerId2, container.GetContainerId());
NUnit.Framework.Assert.IsNull(container.GetContainer().GetContainerToken());
// acquire the container.
IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
NUnit.Framework.Assert.AreEqual(containerId2, containers[0].GetId());
// container token is generated.
NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken());
rm1.Stop();
}
// Test regular RM restart/failover, new RM should not count
// AM failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestRMRestartOrFailoverNotCountedForAMFailures()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
// AM should be restarted even though max-am-attempt is 1.
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt1).MayBeLastAttempt());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
// re-register the NM
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus
>();
status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager);
status.SetContainerId(attempt1.GetMasterContainer().GetId());
status.SetContainerState(ContainerState.Complete);
status.SetDiagnostics(string.Empty);
nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null);
rm2.WaitForState(attempt1.GetAppAttemptId(), RMAppAttemptState.Failed);
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledByResourcemanager, appState
.GetAttempt(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Will automatically start a new AppAttempt in rm2
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt3 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt3.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
public virtual void TestExistenceOfResourceRequestInRMContainer()
{
Configuration conf = new Configuration();
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
RMApp app1 = rm1.SubmitApp(1024);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
ResourceScheduler scheduler = rm1.GetResourceScheduler();
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
// Verify whether list of ResourceRequest is present in RMContainer
// while moving to ALLOCATED state
NUnit.Framework.Assert.IsNotNull(scheduler.GetRMContainer(containerId2).GetResourceRequests
());
// Allocate container
am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
rm1.WaitForState(nm1, containerId2, RMContainerState.Acquired);
// After RMContainer moving to ACQUIRED state, list of ResourceRequest will
// be empty
NUnit.Framework.Assert.IsNull(scheduler.GetRMContainer(containerId2).GetResourceRequests
());
}
// To avoid using cached client
/// <exception cref="System.Exception"/>
public virtual void TestAMRMMasterKeysUpdate()
{
AtomicReference <AMRMTokenSecretManager> spySecretMgrRef = new AtomicReference <AMRMTokenSecretManager
>();
MockRM rm = new _MockRM_349(this, spySecretMgrRef, conf);
// Skip the login.
rm.Start();
MockNM nm = rm.RegisterNode("127.0.0.1:1234", 8000);
RMApp app = rm.SubmitApp(200);
MockAM am = MockRM.LaunchAndRegisterAM(app, rm, nm);
AMRMTokenSecretManager spySecretMgr = spySecretMgrRef.Get();
// Do allocate. Should not update AMRMToken
AllocateResponse response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<AllocateRequest>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> oldToken = rm.GetRMContext
().GetRMApps()[app.GetApplicationId()].GetRMAppAttempt(am.GetApplicationAttemptId
()).GetAMRMToken();
// roll over the master key
// Do allocate again. the AM should get the latest AMRMToken
rm.GetRMContext().GetAMRMTokenSecretManager().RollMasterKey();
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = ConverterUtils
.ConvertFromYarn(response.GetAMRMToken(), new Text(response.GetAMRMToken().GetService
()));
NUnit.Framework.Assert.AreEqual(amrmToken.DecodeIdentifier().GetKeyId(), rm.GetRMContext
().GetAMRMTokenSecretManager().GetMasterKey().GetMasterKey().GetKeyId());
// Do allocate again with the same old token and verify the RM sends
// back the last generated token instead of generating it again.
Org.Mockito.Mockito.Reset(spySecretMgr);
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(am.GetApplicationAttemptId
().ToString(), new string[0]);
ugi.AddTokenIdentifier(oldToken.DecodeIdentifier());
response = am.DoAllocateAs(ugi, Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken());
Org.Mockito.Mockito.Verify(spySecretMgr, Org.Mockito.Mockito.Never()).CreateAndGetAMRMToken
(Matchers.IsA <ApplicationAttemptId>());
// Do allocate again with the updated token and verify we do not
// receive a new token to use.
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
// Activate the next master key. Since there is new master key generated
// in AMRMTokenSecretManager. The AMRMToken will not get updated for AM
rm.GetRMContext().GetAMRMTokenSecretManager().ActivateNextMasterKey();
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestStoreOnlyAMContainerMetrics()
{
Configuration conf = new Configuration();
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
conf.SetBoolean(YarnConfiguration.ApplicationHistorySaveNonAmContainerMetaInfo, false
);
MockRM rm1 = new MockRM(conf);
SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>();
rm1.GetRMContext().SetSystemMetricsPublisher(publisher);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
RMApp app1 = rm1.SubmitApp(1024);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Running);
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
rm1.WaitForState(nm1, containerId2, RMContainerState.Acquired);
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Running);
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Complete);
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
rm1.WaitForState(nm1, containerId2, RMContainerState.Completed);
rm1.Stop();
// RMContainer should be publishing system metrics only for AM container.
Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Times(1)).ContainerCreated
(Matchers.Any <RMContainer>(), Matchers.AnyLong());
Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Times(1)).ContainerFinished
(Matchers.Any <RMContainer>(), Matchers.AnyLong());
}
/// <exception cref="System.Exception"/>
private void TestRMWritingMassiveHistory(MockRM rm)
{
rm.Start();
MockNM nm = rm.RegisterNode("127.0.0.1:1234", 1024 * 10100);
RMApp app = rm.SubmitApp(1024);
nm.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
int request = 10000;
am.Allocate("127.0.0.1", 1024, request, new AList <ContainerId>());
nm.NodeHeartbeat(true);
IList <Container> allocated = am.Allocate(new AList <ResourceRequest>(), new AList <
ContainerId>()).GetAllocatedContainers();
int waitCount = 0;
int allocatedSize = allocated.Count;
while (allocatedSize < request && waitCount++ < 200)
{
Sharpen.Thread.Sleep(300);
allocated = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
allocatedSize += allocated.Count;
nm.NodeHeartbeat(true);
}
NUnit.Framework.Assert.AreEqual(request, allocatedSize);
am.UnregisterAppAttempt();
am.WaitForState(RMAppAttemptState.Finishing);
nm.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
NodeHeartbeatResponse resp = nm.NodeHeartbeat(true);
IList <ContainerId> cleaned = resp.GetContainersToCleanup();
int cleanedSize = cleaned.Count;
waitCount = 0;
while (cleanedSize < allocatedSize && waitCount++ < 200)
{
Sharpen.Thread.Sleep(300);
resp = nm.NodeHeartbeat(true);
cleaned = resp.GetContainersToCleanup();
cleanedSize += cleaned.Count;
}
NUnit.Framework.Assert.AreEqual(allocatedSize, cleanedSize);
rm.WaitForState(app.GetApplicationId(), RMAppState.Finished);
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestRMAppAttemptFailuresValidityInterval()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 2.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 2);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// set window size to a larger number : 20s
// we will verify the app should be failed if
// two continuous attempts failed in 20s.
RMApp app = rm1.SubmitApp(200, 20000);
MockAM am = MockRM.LaunchAM(app, rm1, nm1);
// Fail current attempt normally
nm1.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Failed);
// launch the second attempt
rm1.WaitForState(app.GetApplicationId(), RMAppState.Accepted);
NUnit.Framework.Assert.AreEqual(2, app.GetAppAttempts().Count);
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)app.GetCurrentAppAttempt()).MayBeLastAttempt
());
MockAM am_2 = MockRM.LaunchAndRegisterAM(app, rm1, nm1);
am_2.WaitForState(RMAppAttemptState.Running);
nm1.NodeHeartbeat(am_2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am_2.WaitForState(RMAppAttemptState.Failed);
// current app should be failed.
rm1.WaitForState(app.GetApplicationId(), RMAppState.Failed);
ControlledClock clock = new ControlledClock(new SystemClock());
// set window size to 6s
RMAppImpl app1 = (RMAppImpl)rm1.SubmitApp(200, 6000);
app1.SetSystemClock(clock);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// Fail attempt1 normally
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
am1.WaitForState(RMAppAttemptState.Failed);
// launch the second attempt
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
NUnit.Framework.Assert.AreEqual(2, app1.GetAppAttempts().Count);
RMAppAttempt attempt2 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt2).MayBeLastAttempt());
MockAM am2 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
am2.WaitForState(RMAppAttemptState.Running);
// wait for 6 seconds
clock.SetTime(Runtime.CurrentTimeMillis() + 6 * 1000);
// Fail attempt2 normally
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
// can launch the third attempt successfully
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
NUnit.Framework.Assert.AreEqual(3, app1.GetAppAttempts().Count);
RMAppAttempt attempt3 = app1.GetCurrentAppAttempt();
clock.Reset();
MockAM am3 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
am3.WaitForState(RMAppAttemptState.Running);
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
// re-register the NM
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus
>();
status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager);
status.SetContainerId(attempt3.GetMasterContainer().GetId());
status.SetContainerState(ContainerState.Complete);
status.SetDiagnostics(string.Empty);
nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null);
rm2.WaitForState(attempt3.GetAppAttemptId(), RMAppAttemptState.Failed);
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// Lauch Attempt 4
MockAM am4 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 4, nm1);
// wait for 6 seconds
clock.SetTime(Runtime.CurrentTimeMillis() + 6 * 1000);
// Fail attempt4 normally
nm1.NodeHeartbeat(am4.GetApplicationAttemptId(), 1, ContainerState.Complete);
am4.WaitForState(RMAppAttemptState.Failed);
// can launch the 5th attempt successfully
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
MockAM am5 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 5, nm1);
clock.Reset();
am5.WaitForState(RMAppAttemptState.Running);
// Fail attempt5 normally
nm1.NodeHeartbeat(am5.GetApplicationAttemptId(), 1, ContainerState.Complete);
am5.WaitForState(RMAppAttemptState.Failed);
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Failed);
rm1.Stop();
rm2.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestResourceRequestRestoreWhenRMContainerIsAtAllocated()
{
ConfigureScheduler();
YarnConfiguration conf = GetConf();
MockRM rm1 = new MockRM(conf);
try
{
rm1.Start();
RMApp app1 = rm1.SubmitApp(200, "name", "user", new Dictionary <ApplicationAccessType
, string>(), false, "default", -1, null, "Test", false, true);
MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockNM nm2 = new MockNM("127.0.0.1:2351", 10240, rm1.GetResourceTrackerService());
nm2.RegisterNode();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
int NumContainers = 1;
// allocate NUM_CONTAINERS containers
am1.Allocate("127.0.0.1", 1024, NumContainers, new AList <ContainerId>());
nm1.NodeHeartbeat(true);
// wait for containers to be allocated.
IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
while (containers.Count != NumContainers)
{
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, am1.Allocate(new AList <ResourceRequest>(),
new AList <ContainerId>()).GetAllocatedContainers());
Sharpen.Thread.Sleep(200);
}
// launch the 2nd container, for testing running container transferred.
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Running);
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Running);
// 3rd container is in Allocated state.
am1.Allocate("127.0.0.1", 1024, NumContainers, new AList <ContainerId>());
nm2.NodeHeartbeat(true);
ContainerId containerId3 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 3);
rm1.WaitForContainerAllocated(nm2, containerId3);
rm1.WaitForState(nm2, containerId3, RMContainerState.Allocated);
// NodeManager restart
nm2.RegisterNode();
// NM restart kills all allocated and running containers.
rm1.WaitForState(nm2, containerId3, RMContainerState.Killed);
// The killed RMContainer request should be restored. In successive
// nodeHeartBeats AM should be able to get container allocated.
containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>())
.GetAllocatedContainers();
while (containers.Count != NumContainers)
{
nm2.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, am1.Allocate(new AList <ResourceRequest>(),
new AList <ContainerId>()).GetAllocatedContainers());
Sharpen.Thread.Sleep(200);
}
nm2.NodeHeartbeat(am1.GetApplicationAttemptId(), 4, ContainerState.Running);
ContainerId containerId4 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 4);
rm1.WaitForState(nm2, containerId4, RMContainerState.Running);
}
finally
{
rm1.Stop();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestNodeUpdate()
{
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y", "z"));
// set mapping:
// h1 -> x
// h2 -> y
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x")));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm = new _MockRM_110(this, GetConfigurationWithQueueLabels(conf));
rm.GetRMContext().SetNodeLabelManager(mgr);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 8000);
MockNM nm2 = rm.RegisterNode("h2:1234", 8000);
MockNM nm3 = rm.RegisterNode("h3:1234", 8000);
ContainerId containerId;
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm.SubmitApp(Gb, "app", "user", null, "a");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm, nm3);
// request a container.
am1.Allocate("*", Gb, 1, new AList <ContainerId>(), "x");
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsTrue(rm.WaitForState(nm1, containerId, RMContainerState.
Allocated, 10 * 1000));
// check used resource:
// queue-a used x=1G, ""=1G
CheckUsedResource(rm, "a", 1024, "x");
CheckUsedResource(rm, "a", 1024);
// change h1's label to z, container should be killed
mgr.ReplaceLabelsOnNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("z")));
NUnit.Framework.Assert.IsTrue(rm.WaitForState(nm1, containerId, RMContainerState.
Killed, 10 * 1000));
// check used resource:
// queue-a used x=0G, ""=1G ("" not changed)
CheckUsedResource(rm, "a", 0, "x");
CheckUsedResource(rm, "a", 1024);
// request a container with label = y
am1.Allocate("*", Gb, 1, new AList <ContainerId>(), "y");
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 3);
NUnit.Framework.Assert.IsTrue(rm.WaitForState(nm2, containerId, RMContainerState.
Allocated, 10 * 1000));
// check used resource:
// queue-a used y=1G, ""=1G
CheckUsedResource(rm, "a", 1024, "y");
CheckUsedResource(rm, "a", 1024);
// change h2's label to no label, container should be killed
mgr.ReplaceLabelsOnNode(ImmutableMap.Of(NodeId.NewInstance("h2", 0), CommonNodeLabelsManager
.EmptyStringSet));
NUnit.Framework.Assert.IsTrue(rm.WaitForState(nm1, containerId, RMContainerState.
Killed, 10 * 1000));
// check used resource:
// queue-a used x=0G, y=0G, ""=1G ("" not changed)
CheckUsedResource(rm, "a", 0, "x");
CheckUsedResource(rm, "a", 0, "y");
CheckUsedResource(rm, "a", 1024);
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 1);
// change h3's label to z, AM container should be killed
mgr.ReplaceLabelsOnNode(ImmutableMap.Of(NodeId.NewInstance("h3", 0), ToSet("z")));
NUnit.Framework.Assert.IsTrue(rm.WaitForState(nm1, containerId, RMContainerState.
Killed, 10 * 1000));
// check used resource:
// queue-a used x=0G, y=0G, ""=1G ("" not changed)
CheckUsedResource(rm, "a", 0, "x");
CheckUsedResource(rm, "a", 0, "y");
CheckUsedResource(rm, "a", 0);
rm.Close();
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocateWithComplexLabels()
{
/*
* Queue structure:
* root (*)
* ________________
* / \
* a x(100%), y(50%) b y(50%), z(100%)
* ________________ ______________
* / / \
* a1 (x,y) b1(no) b2(y,z)
* 100% y = 100%, z = 100%
*
* Node structure:
* h1 : x
* h2 : y
* h3 : y
* h4 : z
* h5 : NO
*
* Total resource:
* x: 4G
* y: 6G
* z: 2G
* *: 2G
*
* Resource of
* a1: x=4G, y=3G, NO=0.2G
* b1: NO=0.9G (max=1G)
* b2: y=3, z=2G, NO=0.9G (max=1G)
*
* Each node can only allocate two containers
*/
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y", "z"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y"), NodeId.NewInstance("h3", 0), ToSet("y"), NodeId
.NewInstance("h4", 0), ToSet("z"), NodeId.NewInstance("h5", 0), RMNodeLabelsManager
.EmptyStringSet));
// inject node label manager
MockRM rm1 = new _MockRM_557(this, GetComplexConfigurationWithQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 2048);
MockNM nm2 = rm1.RegisterNode("h2:1234", 2048);
MockNM nm3 = rm1.RegisterNode("h3:1234", 2048);
MockNM nm4 = rm1.RegisterNode("h4:1234", 2048);
MockNM nm5 = rm1.RegisterNode("h5:1234", 2048);
ContainerId containerId;
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(1024, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// request a container (label = y). can be allocated on nm2
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "y");
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2L);
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h2");
// launch an app to queue b1 (label = y), and check all container will
// be allocated in h5
RMApp app2 = rm1.SubmitApp(1024, "app", "user", null, "b1");
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm5);
// request a container for AM, will succeed
// and now b1's queue capacity will be used, cannot allocate more containers
// (Maximum capacity reached)
am2.Allocate("*", 1024, 1, new AList <ContainerId>());
containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm4, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm5, containerId, RMContainerState
.Allocated, 10 * 1000));
// launch an app to queue b2
RMApp app3 = rm1.SubmitApp(1024, "app", "user", null, "b2");
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm5);
// request a container. try to allocate on nm1 (label = x) and nm3 (label =
// y,z). Will successfully allocate on nm3
am3.Allocate("*", 1024, 1, new AList <ContainerId>(), "y");
containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3");
// try to allocate container (request label = z) on nm4 (label = y,z).
// Will successfully allocate on nm4 only.
am3.Allocate("*", 1024, 1, new AList <ContainerId>(), "z");
containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 3L);
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm4, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h4");
rm1.Close();
}
// AM container preempted, nm disk failure
// should not be counted towards AM max retry count.
/// <exception cref="System.Exception"/>
public virtual void TestShouldNotCountFailureToMaxAttemptRetry()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler();
ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 1);
// Preempt the first attempt;
scheduler.KillContainer(scheduler.GetRMContainer(amContainer));
am1.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry());
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
// AM should be restarted even though max-am-attempt is 1.
MockAM am2 = rm1.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
RMAppAttempt attempt2 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt2).MayBeLastAttempt());
// Preempt the second attempt.
ContainerId amContainer2 = ContainerId.NewContainerId(am2.GetApplicationAttemptId
(), 1);
scheduler.KillContainer(scheduler.GetRMContainer(amContainer2));
am2.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(!attempt2.ShouldCountTowardsMaxAttemptRetry());
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
MockAM am3 = rm1.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 3, nm1);
RMAppAttempt attempt3 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt3).MayBeLastAttempt());
// mimic NM disk_failure
ContainerStatus containerStatus = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerStatus
>();
containerStatus.SetContainerId(attempt3.GetMasterContainer().GetId());
containerStatus.SetDiagnostics("mimic NM disk_failure");
containerStatus.SetState(ContainerState.Complete);
containerStatus.SetExitStatus(ContainerExitStatus.DisksFailed);
IDictionary <ApplicationId, IList <ContainerStatus> > conts = new Dictionary <ApplicationId
, IList <ContainerStatus> >();
conts[app1.GetApplicationId()] = Sharpen.Collections.SingletonList(containerStatus
);
nm1.NodeHeartbeat(conts, true);
am3.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(!attempt3.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.DisksFailed, appState.GetAttempt
(am3.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
MockAM am4 = rm1.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 4, nm1);
RMAppAttempt attempt4 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt4).MayBeLastAttempt());
// create second NM, and register to rm1
MockNM nm2 = new MockNM("127.0.0.1:2234", 8000, rm1.GetResourceTrackerService());
nm2.RegisterNode();
// nm1 heartbeats to report unhealthy
// This will mimic ContainerExitStatus.ABORT
nm1.NodeHeartbeat(false);
am4.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(!attempt4.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, appState.GetAttempt(
am4.GetApplicationAttemptId()).GetAMContainerExitStatus());
// launch next AM in nm2
nm2.NodeHeartbeat(true);
MockAM am5 = rm1.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 5, nm2);
RMAppAttempt attempt5 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt5).MayBeLastAttempt());
// fail the AM normally
nm2.NodeHeartbeat(am5.GetApplicationAttemptId(), 1, ContainerState.Complete);
am5.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(attempt5.ShouldCountTowardsMaxAttemptRetry());
// AM should not be restarted.
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Failed);
NUnit.Framework.Assert.AreEqual(5, app1.GetAppAttempts().Count);
rm1.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestNMTokensRebindOnAMRestart()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 3);
MockRM rm1 = new MockRM(conf);
rm1.Start();
RMApp app1 = rm1.SubmitApp(200, "myname", "myuser", new Dictionary <ApplicationAccessType
, string>(), false, "default", -1, null, "MAPREDUCE", false, true);
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockNM nm2 = new MockNM("127.1.1.1:4321", 8000, rm1.GetResourceTrackerService());
nm2.RegisterNode();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
IList <Container> containers = new AList <Container>();
// nmTokens keeps track of all the nmTokens issued in the allocate call.
IList <NMToken> expectedNMTokens = new AList <NMToken>();
// am1 allocate 2 container on nm1.
// first container
while (true)
{
AllocateResponse response = am1.Allocate("127.0.0.1", 2000, 2, new AList <ContainerId
>());
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
if (containers.Count == 2)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
// launch the container-2
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Running);
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Running);
// launch the container-3
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 3, ContainerState.Running);
ContainerId containerId3 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 3);
rm1.WaitForState(nm1, containerId3, RMContainerState.Running);
// fail am1
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
am1.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// restart the am
MockAM am2 = MockRM.LaunchAM(app1, rm1, nm1);
RegisterApplicationMasterResponse registerResponse = am2.RegisterAppAttempt();
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Running);
// check am2 get the nm token from am1.
NUnit.Framework.Assert.AreEqual(expectedNMTokens, registerResponse.GetNMTokensFromPreviousAttempts
());
// am2 allocate 1 container on nm2
containers = new AList <Container>();
while (true)
{
AllocateResponse allocateResponse = am2.Allocate("127.1.1.1", 4000, 1, new AList <
ContainerId>());
nm2.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, allocateResponse.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, allocateResponse.GetNMTokens());
if (containers.Count == 1)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 2, ContainerState.Running);
ContainerId am2ContainerId2 = ContainerId.NewContainerId(am2.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, am2ContainerId2, RMContainerState.Running);
// fail am2.
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// restart am
MockAM am3 = MockRM.LaunchAM(app1, rm1, nm1);
registerResponse = am3.RegisterAppAttempt();
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Running);
// check am3 get the NM token from both am1 and am2;
IList <NMToken> transferredTokens = registerResponse.GetNMTokensFromPreviousAttempts
();
NUnit.Framework.Assert.AreEqual(2, transferredTokens.Count);
NUnit.Framework.Assert.IsTrue(transferredTokens.ContainsAll(expectedNMTokens));
rm1.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestAMRestartWithExistingContainers()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 2);
MockRM rm1 = new MockRM(conf);
rm1.Start();
RMApp app1 = rm1.SubmitApp(200, "name", "user", new Dictionary <ApplicationAccessType
, string>(), false, "default", -1, null, "MAPREDUCE", false, true);
MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockNM nm2 = new MockNM("127.0.0.1:2351", 4089, rm1.GetResourceTrackerService());
nm2.RegisterNode();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
int NumContainers = 3;
// allocate NUM_CONTAINERS containers
am1.Allocate("127.0.0.1", 1024, NumContainers, new AList <ContainerId>());
nm1.NodeHeartbeat(true);
// wait for containers to be allocated.
IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
while (containers.Count != NumContainers)
{
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, am1.Allocate(new AList <ResourceRequest>(),
new AList <ContainerId>()).GetAllocatedContainers());
Sharpen.Thread.Sleep(200);
}
// launch the 2nd container, for testing running container transferred.
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Running);
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Running);
// launch the 3rd container, for testing container allocated by previous
// attempt is completed by the next new attempt/
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 3, ContainerState.Running);
ContainerId containerId3 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 3);
rm1.WaitForState(nm1, containerId3, RMContainerState.Running);
// 4th container still in AQUIRED state. for testing Acquired container is
// always killed.
ContainerId containerId4 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 4);
rm1.WaitForState(nm1, containerId4, RMContainerState.Acquired);
// 5th container is in Allocated state. for testing allocated container is
// always killed.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
nm1.NodeHeartbeat(true);
ContainerId containerId5 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 5);
rm1.WaitForContainerAllocated(nm1, containerId5);
rm1.WaitForState(nm1, containerId5, RMContainerState.Allocated);
// 6th container is in Reserved state.
am1.Allocate("127.0.0.1", 6000, 1, new AList <ContainerId>());
ContainerId containerId6 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 6);
nm1.NodeHeartbeat(true);
SchedulerApplicationAttempt schedulerAttempt = ((AbstractYarnScheduler)rm1.GetResourceScheduler
()).GetCurrentAttemptForContainer(containerId6);
while (schedulerAttempt.GetReservedContainers().IsEmpty())
{
System.Console.Out.WriteLine("Waiting for container " + containerId6 + " to be reserved."
);
nm1.NodeHeartbeat(true);
Sharpen.Thread.Sleep(200);
}
// assert containerId6 is reserved.
NUnit.Framework.Assert.AreEqual(containerId6, schedulerAttempt.GetReservedContainers
()[0].GetContainerId());
// fail the AM by sending CONTAINER_FINISHED event without registering.
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
am1.WaitForState(RMAppAttemptState.Failed);
// wait for some time. previous AM's running containers should still remain
// in scheduler even though am failed
Sharpen.Thread.Sleep(3000);
rm1.WaitForState(nm1, containerId2, RMContainerState.Running);
// acquired/allocated containers are cleaned up.
NUnit.Framework.Assert.IsNull(rm1.GetResourceScheduler().GetRMContainer(containerId4
));
NUnit.Framework.Assert.IsNull(rm1.GetResourceScheduler().GetRMContainer(containerId5
));
// wait for app to start a new attempt.
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// assert this is a new AM.
ApplicationAttemptId newAttemptId = app1.GetCurrentAppAttempt().GetAppAttemptId();
NUnit.Framework.Assert.IsFalse(newAttemptId.Equals(am1.GetApplicationAttemptId())
);
// launch the new AM
RMAppAttempt attempt2 = app1.GetCurrentAppAttempt();
nm1.NodeHeartbeat(true);
MockAM am2 = rm1.SendAMLaunched(attempt2.GetAppAttemptId());
RegisterApplicationMasterResponse registerResponse = am2.RegisterAppAttempt();
// Assert two containers are running: container2 and container3;
NUnit.Framework.Assert.AreEqual(2, registerResponse.GetContainersFromPreviousAttempts
().Count);
bool containerId2Exists = false;
bool containerId3Exists = false;
foreach (Container container in registerResponse.GetContainersFromPreviousAttempts
())
{
if (container.GetId().Equals(containerId2))
{
containerId2Exists = true;
}
if (container.GetId().Equals(containerId3))
{
containerId3Exists = true;
}
}
NUnit.Framework.Assert.IsTrue(containerId2Exists && containerId3Exists);
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Running);
// complete container by sending the container complete event which has earlier
// attempt's attemptId
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 3, ContainerState.Complete);
// Even though the completed container containerId3 event was sent to the
// earlier failed attempt, new RMAppAttempt can also capture this container
// info.
// completed containerId4 is also transferred to the new attempt.
RMAppAttempt newAttempt = app1.GetRMAppAttempt(am2.GetApplicationAttemptId());
// 4 containers finished, acquired/allocated/reserved/completed.
WaitForContainersToFinish(4, newAttempt);
bool container3Exists = false;
bool container4Exists = false;
bool container5Exists = false;
bool container6Exists = false;
foreach (ContainerStatus status in newAttempt.GetJustFinishedContainers())
{
if (status.GetContainerId().Equals(containerId3))
{
// containerId3 is the container ran by previous attempt but finished by the
// new attempt.
container3Exists = true;
}
if (status.GetContainerId().Equals(containerId4))
{
// containerId4 is the Acquired Container killed by the previous attempt,
// it's now inside new attempt's finished container list.
container4Exists = true;
}
if (status.GetContainerId().Equals(containerId5))
{
// containerId5 is the Allocated container killed by previous failed attempt.
container5Exists = true;
}
if (status.GetContainerId().Equals(containerId6))
{
// containerId6 is the reserved container killed by previous failed attempt.
container6Exists = true;
}
}
NUnit.Framework.Assert.IsTrue(container3Exists && container4Exists && container5Exists &&
container6Exists);
// New SchedulerApplicationAttempt also has the containers info.
rm1.WaitForState(nm1, containerId2, RMContainerState.Running);
// record the scheduler attempt for testing.
SchedulerApplicationAttempt schedulerNewAttempt = ((AbstractYarnScheduler)rm1.GetResourceScheduler
()).GetCurrentAttemptForContainer(containerId2);
// finish this application
MockRM.FinishAMAndVerifyAppState(app1, rm1, nm1, am2);
// the 2nd attempt released the 1st attempt's running container, when the
// 2nd attempt finishes.
NUnit.Framework.Assert.IsFalse(schedulerNewAttempt.GetLiveContainers().Contains(containerId2
));
// all 4 normal containers finished.
System.Console.Out.WriteLine("New attempt's just finished containers: " + newAttempt
.GetJustFinishedContainers());
WaitForContainersToFinish(5, newAttempt);
rm1.Stop();
}