/// <exception cref="System.Exception"/>
public virtual void TestAppCleanupWhenNMReconnects()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// start RM
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// create app and launch the AM
RMApp app0 = rm1.SubmitApp(200);
MockAM am0 = LaunchAM(app0, rm1, nm1);
nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Complete);
rm1.WaitForState(app0.GetApplicationId(), RMAppState.Failed);
// wait for application cleanup message received
WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
// reconnect NM with application still active
nm1.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
rm1.Stop();
}
/// <exception cref="System.Exception"/>
private void TestMinimumAllocation(YarnConfiguration conf, int testAlloc)
{
MockRM rm = new MockRM(conf);
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(testAlloc);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId
());
int checkAlloc = conf.GetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, YarnConfiguration
.DefaultRmSchedulerMinimumAllocationMb);
NUnit.Framework.Assert.AreEqual(checkAlloc, report_nm1.GetUsedResource().GetMemory
());
rm.Stop();
}
/// <exception cref="System.Exception"/>
protected internal virtual void AllocateContainersAndValidateNMTokens(MockAM am,
AList <Container> containersReceived, int totalContainerRequested, Dictionary <string
, Token> nmTokens, MockNM nm)
{
AList <ContainerId> releaseContainerList = new AList <ContainerId>();
AllocateResponse response;
AList <ResourceRequest> resourceRequest = new AList <ResourceRequest>();
while (containersReceived.Count < totalContainerRequested)
{
nm.NodeHeartbeat(true);
Log.Info("requesting containers..");
response = am.Allocate(resourceRequest, releaseContainerList);
Sharpen.Collections.AddAll(containersReceived, response.GetAllocatedContainers());
if (!response.GetNMTokens().IsEmpty())
{
foreach (NMToken nmToken in response.GetNMTokens())
{
string nodeId = nmToken.GetNodeId().ToString();
if (nmTokens.Contains(nodeId))
{
NUnit.Framework.Assert.Fail("Duplicate NMToken received for : " + nodeId);
}
nmTokens[nodeId] = nmToken.GetToken();
}
}
Log.Info("Got " + containersReceived.Count + " containers. Waiting to get " + totalContainerRequested
);
Sharpen.Thread.Sleep(WaitSleepMs);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidatedAMHostPortOnAMRestart()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// a failed app
RMApp app2 = rm1.SubmitApp(200);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app2.GetApplicationId(), RMAppState.Accepted);
// before new attempt is launched, the app report returns the invalid AM
// host and port.
GetApplicationReportRequest request1 = GetApplicationReportRequest.NewInstance(app2
.GetApplicationId());
ApplicationReport report1 = rm1.GetClientRMService().GetApplicationReport(request1
).GetApplicationReport();
NUnit.Framework.Assert.AreEqual("N/A", report1.GetHost());
NUnit.Framework.Assert.AreEqual(-1, report1.GetRpcPort());
}
/// <exception cref="System.Exception"/>
public virtual void SendAMLaunchFailed(ApplicationAttemptId appAttemptId)
{
MockAM am = new MockAM(GetRMContext(), masterService, appAttemptId);
am.WaitForState(RMAppAttemptState.Allocated);
GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptEvent(appAttemptId
, RMAppAttemptEventType.LaunchFailed, "Failed"));
}
/// <exception cref="System.Exception"/>
public virtual void TestAppOnMultiNode()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
conf.Set("yarn.scheduler.capacity.node-locality-delay", "-1");
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
MockNM nm2 = rm.RegisterNode("h2:5678", 10240);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
//request for containers
int request = 13;
am.Allocate("h1", 1000, request, new AList <ContainerId>());
//kick the scheduler
IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
>()).GetAllocatedContainers();
int contReceived = conts.Count;
while (contReceived < 3)
{
//only 3 containers are available on node1
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 3);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(3, conts.Count);
//send node2 heartbeat
conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
contReceived = conts.Count;
while (contReceived < 10)
{
nm2.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 10);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(10, conts.Count);
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidContainerReleaseRequest()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(1024);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler
nm1.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(1000);
alloc1Response = am1.Schedule();
}
NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0);
RMApp app2 = rm.SubmitApp(1024);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId());
am2.RegisterAppAttempt();
// Now trying to release container allocated for app1 -> appAttempt1.
ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId();
am2.AddContainerToBeReleased(cId);
try
{
am2.Schedule();
NUnit.Framework.Assert.Fail("Exception was expected!!");
}
catch (InvalidContainerReleaseException e)
{
StringBuilder sb = new StringBuilder("Cannot release container : ");
sb.Append(cId.ToString());
sb.Append(" not belonging to this application attempt : ");
sb.Append(attempt2.GetAppAttemptId().ToString());
NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString()));
}
}
finally
{
if (rm != null)
{
rm.Stop();
}
}
}
/// <exception cref="System.Exception"/>
public static MockAM LaunchAndRegisterAM(RMApp app, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
rm, MockNM nm)
{
MockAM am = LaunchAM(app, rm, nm);
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
return(am);
}
public virtual void TestAMLaunchAndCleanup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
TestApplicationMasterLauncher.MyContainerManagerImpl containerManager = new TestApplicationMasterLauncher.MyContainerManagerImpl
();
MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
// kick the scheduling
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.launched == false && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue(containerManager.launched);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId appAttemptId = attempt.GetAppAttemptId();
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), containerManager.attemptIdAtContainerManager
);
NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), containerManager.submitTimeAtContainerManager
);
NUnit.Framework.Assert.AreEqual(app.GetRMAppAttempt(appAttemptId).GetMasterContainer
().GetId().ToString(), containerManager.containerIdAtContainerManager);
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId().ToString(), containerManager.nmHostAtContainerManager
);
NUnit.Framework.Assert.AreEqual(YarnConfiguration.DefaultRmAmMaxAttempts, containerManager
.maxAppAttempts);
MockAM am = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), appAttemptId
);
am.RegisterAppAttempt();
am.UnregisterAppAttempt();
//complete the AM container to finish the app normally
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
waitCount = 0;
while (containerManager.cleanedup == false && waitCount++ < 20)
{
Log.Info("Waiting for AM Cleanup to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue(containerManager.cleanedup);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
/// <exception cref="System.Exception"/>
private MockAM LaunchAM(RMApp app, MockRM rm, MockNM nm)
{
RMAppAttempt attempt = app.GetCurrentAppAttempt();
nm.NodeHeartbeat(true);
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
return(am);
}
public static IList <NMContainerStatus> CreateNMContainerStatusForApp(MockAM am)
{
IList <NMContainerStatus> list = new AList <NMContainerStatus>();
NMContainerStatus amContainer = CreateNMContainerStatus(am.GetApplicationAttemptId
(), 1, ContainerState.Running, 1024);
NMContainerStatus completedContainer = CreateNMContainerStatus(am.GetApplicationAttemptId
(), 2, ContainerState.Complete, 2048);
list.AddItem(amContainer);
list.AddItem(completedContainer);
return(list);
}
// Disable webapp
/// <exception cref="System.Exception"/>
public static void FinishAMAndVerifyAppState(RMApp rmApp, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
rm, MockNM nm, MockAM am)
{
FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
.Succeeded, string.Empty, string.Empty);
am.UnregisterAppAttempt(req, true);
am.WaitForState(RMAppAttemptState.Finishing);
nm.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.WaitForState(rmApp.GetApplicationId(), RMAppState.Finished);
}
// This is to test AM Host and rpc port are invalidated after the am attempt
// is killed or failed, so that client doesn't get the wrong information.
/// <exception cref="System.Exception"/>
public virtual void TestInvalidateAMHostPortWhenAMFailedOrKilled()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MockRM rm1 = new MockRM(conf);
rm1.Start();
// a succeeded app
RMApp app1 = rm1.SubmitApp(200);
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm1, nm1, am1);
// a failed app
RMApp app2 = rm1.SubmitApp(200);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app2.GetApplicationId(), RMAppState.Failed);
// a killed app
RMApp app3 = rm1.SubmitApp(200);
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm1);
rm1.KillApp(app3.GetApplicationId());
rm1.WaitForState(app3.GetApplicationId(), RMAppState.Killed);
rm1.WaitForState(am3.GetApplicationAttemptId(), RMAppAttemptState.Killed);
GetApplicationsRequest request1 = GetApplicationsRequest.NewInstance(EnumSet.Of(YarnApplicationState
.Finished, YarnApplicationState.Killed, YarnApplicationState.Failed));
GetApplicationsResponse response1 = rm1.GetClientRMService().GetApplications(request1
);
IList <ApplicationReport> appList1 = response1.GetApplicationList();
NUnit.Framework.Assert.AreEqual(3, appList1.Count);
foreach (ApplicationReport report in appList1)
{
// killed/failed apps host and rpc port are invalidated.
if (report.GetApplicationId().Equals(app2.GetApplicationId()) || report.GetApplicationId
().Equals(app3.GetApplicationId()))
{
NUnit.Framework.Assert.AreEqual("N/A", report.GetHost());
NUnit.Framework.Assert.AreEqual(-1, report.GetRpcPort());
}
// succeeded app's host and rpc port is not invalidated
if (report.GetApplicationId().Equals(app1.GetApplicationId()))
{
NUnit.Framework.Assert.IsFalse(report.GetHost().Equals("N/A"));
NUnit.Framework.Assert.IsTrue(report.GetRpcPort() != -1);
}
}
}
// Test even if AM container is allocated with containerId not equal to 1, the
// following allocate requests from AM should be able to retrieve the
// corresponding NM Token.
/// <exception cref="System.Exception"/>
public virtual void TestNMTokenSentForNormalContainer()
{
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName
());
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
// Call getNewContainerId to increase container Id so that the AM container
// Id doesn't equal to one.
CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId();
// kick the scheduling
nm1.NodeHeartbeat(true);
MockAM am = MockRM.LaunchAM(app, rm, nm1);
// am container Id not equal to 1.
NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId
() != 1);
// NMSecretManager doesn't record the node on which the am is allocated.
NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
int NumContainers = 1;
IList <Container> containers = new AList <Container>();
// nmTokens keeps track of all the nmTokens issued in the allocate call.
IList <NMToken> expectedNMTokens = new AList <NMToken>();
// am1 allocate 1 container on nm1.
while (true)
{
AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList
<ContainerId>());
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
if (containers.Count == NumContainers)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
NodeId nodeId = expectedNMTokens[0].GetNodeId();
// NMToken is sent for the allocated container.
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId);
}
/// <exception cref="System.Exception"/>
public static MockAM LaunchAM(RMApp app, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
rm, MockNM nm)
{
rm.WaitForState(app.GetApplicationId(), RMAppState.Accepted);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
System.Console.Out.WriteLine("Launch AM " + attempt.GetAppAttemptId());
nm.NodeHeartbeat(true);
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
rm.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Launched);
return(am);
}
/// <exception cref="System.Exception"/>
public virtual void TestAppCleanupWhenRMRestartedBeforeAppFinished()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// start RM
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 1024, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockNM nm2 = new MockNM("127.0.0.1:5678", 1024, rm1.GetResourceTrackerService());
nm2.RegisterNode();
// create app and launch the AM
RMApp app0 = rm1.SubmitApp(200);
MockAM am0 = LaunchAM(app0, rm1, nm1);
// alloc another container on nm2
AllocateResponse allocResponse = am0.Allocate(Arrays.AsList(ResourceRequest.NewInstance
(Priority.NewInstance(1), "*", Resource.NewInstance(1024, 0), 1)), null);
while (null == allocResponse.GetAllocatedContainers() || allocResponse.GetAllocatedContainers
().IsEmpty())
{
nm2.NodeHeartbeat(true);
allocResponse = am0.Allocate(null, null);
Sharpen.Thread.Sleep(1000);
}
// start new RM
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
// nm1/nm2 register to rm2, and do a heartbeat
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode(Arrays.AsList(NMContainerStatus.NewInstance(ContainerId.NewContainerId
(am0.GetApplicationAttemptId(), 1), ContainerState.Complete, Resource.NewInstance
(1024, 1), string.Empty, 0, Priority.NewInstance(0), 1234)), Arrays.AsList(app0.
GetApplicationId()));
nm2.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm2.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
// assert app state has been saved.
rm2.WaitForState(app0.GetApplicationId(), RMAppState.Failed);
// wait for application cleanup message received on NM1
WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
// wait for application cleanup message received on NM2
WaitForAppCleanupMessageRecved(nm2, app0.GetApplicationId());
rm1.Stop();
rm2.Stop();
}
// from AMLauncher
/// <exception cref="System.Exception"/>
public virtual MockAM SendAMLaunched(ApplicationAttemptId appAttemptId)
{
MockAM am = new MockAM(GetRMContext(), masterService, appAttemptId);
am.WaitForState(RMAppAttemptState.Allocated);
//create and set AMRMToken
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = this.rmContext
.GetAMRMTokenSecretManager().CreateAndGetAMRMToken(appAttemptId);
((RMAppAttemptImpl)this.rmContext.GetRMApps()[appAttemptId.GetApplicationId()].GetRMAppAttempt
(appAttemptId)).SetAMRMToken(amrmToken);
GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppAttemptEvent(appAttemptId
, RMAppAttemptEventType.Launched));
return(am);
}
/// <exception cref="System.Exception"/>
public virtual void TestUsageWithOneAttemptAndOneContainer()
{
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());
nm.RegisterNode();
RMApp app0 = rm.SubmitApp(200);
RMAppMetrics rmAppMetrics = app0.GetRMAppMetrics();
NUnit.Framework.Assert.IsTrue("Before app submittion, memory seconds should have been 0 but was "
+ rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() == 0);
NUnit.Framework.Assert.IsTrue("Before app submission, vcore seconds should have been 0 but was "
+ rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() == 0);
RMAppAttempt attempt0 = app0.GetCurrentAppAttempt();
nm.NodeHeartbeat(true);
MockAM am0 = rm.SendAMLaunched(attempt0.GetAppAttemptId());
am0.RegisterAppAttempt();
RMContainer rmContainer = rm.GetResourceScheduler().GetRMContainer(attempt0.GetMasterContainer
().GetId());
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
while (rmAppMetrics.GetMemorySeconds() <= 0 && cumulativeSleepTime < 5000)
{
Sharpen.Thread.Sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
}
rmAppMetrics = app0.GetRMAppMetrics();
NUnit.Framework.Assert.IsTrue("While app is running, memory seconds should be >0 but is "
+ rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() > 0);
NUnit.Framework.Assert.IsTrue("While app is running, vcore seconds should be >0 but is "
+ rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() > 0);
MockRM.FinishAMAndVerifyAppState(app0, rm, nm, am0);
AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(rmContainer);
rmAppMetrics = app0.GetRMAppMetrics();
NUnit.Framework.Assert.AreEqual("Unexcpected MemorySeconds value", ru.GetMemorySeconds
(), rmAppMetrics.GetMemorySeconds());
NUnit.Framework.Assert.AreEqual("Unexpected VcoreSeconds value", ru.GetVcoreSeconds
(), rmAppMetrics.GetVcoreSeconds());
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestResourceTypes()
{
Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes> >
driver = new Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
> >();
CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();
csconf.SetResourceComparator(typeof(DominantResourceCalculator));
YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf);
testCapacityDRConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
), typeof(ResourceScheduler));
YarnConfiguration testCapacityDefConf = new YarnConfiguration();
testCapacityDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
), typeof(ResourceScheduler));
YarnConfiguration testFairDefConf = new YarnConfiguration();
testFairDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(FairScheduler), typeof(
ResourceScheduler));
driver[conf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory);
driver[testCapacityDRConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.
Cpu, YarnServiceProtos.SchedulerResourceTypes.Memory);
driver[testCapacityDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes
.Memory);
driver[testFairDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory
, YarnServiceProtos.SchedulerResourceTypes.Cpu);
foreach (KeyValuePair <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
> > entry in driver)
{
EnumSet <YarnServiceProtos.SchedulerResourceTypes> expectedValue = entry.Value;
MockRM rm = new MockRM(entry.Key);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
RMApp app1 = rm.SubmitApp(2048);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
RegisterApplicationMasterResponse resp = am1.RegisterAppAttempt();
EnumSet <YarnServiceProtos.SchedulerResourceTypes> types = resp.GetSchedulerResourceTypes
();
Log.Info("types = " + types.ToString());
NUnit.Framework.Assert.AreEqual(expectedValue, types);
rm.Stop();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestFinishApplicationMasterBeforeRegistering()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(2048);
MockAM am1 = MockRM.LaunchAM(app1, rm, nm1);
FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
.Failed, string.Empty, string.Empty);
try
{
am1.UnregisterAppAttempt(req, false);
NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
);
}
catch (ApplicationMasterNotRegisteredException e)
{
NUnit.Framework.Assert.IsNotNull(e);
NUnit.Framework.Assert.IsNotNull(e.Message);
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Application Master is trying to unregister before registering for:"
));
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
);
}
am1.RegisterAppAttempt();
am1.UnregisterAppAttempt(req, false);
am1.WaitForState(RMAppAttemptState.Finishing);
}
finally
{
if (rm != null)
{
rm.Stop();
}
}
}
/// <summary>Validate killing an application when it is at accepted state.</summary>
/// <exception cref="System.Exception">exception</exception>
public virtual void TestApplicationKillAtAcceptedState()
{
Dispatcher dispatcher = new _AsyncDispatcher_573();
MockRM rm = new _MockRM_596(dispatcher, conf);
// test metrics
QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
int appsKilled = metrics.GetAppsKilled();
int appsSubmitted = metrics.GetAppsSubmitted();
rm.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());
nm1.RegisterNode();
// a failed app
RMApp application = rm.SubmitApp(200);
MockAM am = MockRM.LaunchAM(application, rm, nm1);
am.WaitForState(RMAppAttemptState.Launched);
nm1.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Running);
rm.WaitForState(application.GetApplicationId(), RMAppState.Accepted);
// Now kill the application before new attempt is launched, the app report
// returns the invalid AM host and port.
KillApplicationRequest request = KillApplicationRequest.NewInstance(application.GetApplicationId
());
rm.GetClientRMService().ForceKillApplication(request);
// Specific test for YARN-1689 follows
// Now let's say a race causes AM to register now. This should not crash RM.
am.RegisterAppAttempt(false);
// We explicitly intercepted the kill-event to RMAppAttempt, so app should
// still be in KILLING state.
rm.WaitForState(application.GetApplicationId(), RMAppState.Killing);
// AM should now be in running
rm.WaitForState(am.GetApplicationAttemptId(), RMAppAttemptState.Running);
// Simulate that appAttempt is killed.
rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppEvent(application
.GetApplicationId(), RMAppEventType.AttemptKilled));
rm.WaitForState(application.GetApplicationId(), RMAppState.Killed);
// test metrics
metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
NUnit.Framework.Assert.AreEqual(appsKilled + 1, metrics.GetAppsKilled());
NUnit.Framework.Assert.AreEqual(appsSubmitted + 1, metrics.GetAppsSubmitted());
}
/// <exception cref="System.Exception"/>
public virtual void TestKillAppWhenFailoverHappensAtRunningState()
{
StartRMs();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// create app and launch the AM
RMApp app0 = rm1.SubmitApp(200);
MockAM am0 = LaunchAM(app0, rm1, nm1);
// failover and kill application
// The application is at RUNNING State when failOver happens.
// Since RMStateStore has already saved ApplicationState, the active RM
// will load the ApplicationState. After that, the application will be at
// ACCEPTED State. Because the application is not at Final State,
// KillApplicationResponse.getIsKillCompleted is expected to return false.
FailOverAndKillApp(app0.GetApplicationId(), am0.GetApplicationAttemptId(), RMAppState
.Running, RMAppAttemptState.Running, RMAppState.Accepted);
}
// Test Kill an app while the app is failing
/// <exception cref="System.Exception"/>
public virtual void TestKillFailingApp()
{
// this dispatcher ignores RMAppAttemptEventType.KILL event
Dispatcher dispatcher = new _AsyncDispatcher_708();
MockRM rm1 = new _MockRM_731(dispatcher, conf);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
rm1.KillApp(app1.GetApplicationId());
// fail the app by sending container_finished event.
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
rm1.WaitForState(am1.GetApplicationAttemptId(), RMAppAttemptState.Failed);
// app is killed, not launching a new attempt
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Killed);
}
/// <exception cref="System.Exception"/>
public virtual void TestAppWithNoContainers()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestKillAppWhenFailOverHappensDuringApplicationKill()
{
// create a customized ClientRMService
// When receives the killApplicationRequest, simply return the response
// and make sure the application will not be KILLED State
StartRMsWithCustomizedClientRMService();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// create app and launch the AM
RMApp app0 = rm1.SubmitApp(200);
MockAM am0 = LaunchAM(app0, rm1, nm1);
// ensure that the app is in running state
NUnit.Framework.Assert.AreEqual(app0.GetState(), RMAppState.Running);
// kill the app.
rm1.KillApp(app0.GetApplicationId());
// failover happens before this application goes to final state.
// The RMAppState that will be loaded by the active rm
// should be ACCEPTED.
FailOverAndKillApp(app0.GetApplicationId(), am0.GetApplicationAttemptId(), RMAppState
.Running, RMAppAttemptState.Running, RMAppState.Accepted);
}
/// <exception cref="System.Exception"/>
public virtual void TestKillAppWhenFailoverHappensAtFinalState()
{
StartRMs();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// create app and launch the AM
RMApp app0 = rm1.SubmitApp(200);
MockAM am0 = LaunchAM(app0, rm1, nm1);
// kill the app.
rm1.KillApp(app0.GetApplicationId());
rm1.WaitForState(app0.GetApplicationId(), RMAppState.Killed);
rm1.WaitForState(am0.GetApplicationAttemptId(), RMAppAttemptState.Killed);
// failover and kill application
// The application is at Killed State and RMStateStore has already
// saved this applicationState. After failover happens, the current
// active RM will load the ApplicationState whose RMAppState is killed.
// Because this application is at Final State,
// KillApplicationResponse.getIsKillCompleted is expected to return true.
FailOverAndKillApp(app0.GetApplicationId(), am0.GetApplicationAttemptId(), RMAppState
.Killed, RMAppAttemptState.Killed, RMAppState.Killed);
}
// Test Kill an app while the app is finishing in the meanwhile.
/// <exception cref="System.Exception"/>
public virtual void TestKillFinishingApp()
{
// this dispatcher ignores RMAppAttemptEventType.KILL event
Dispatcher dispatcher = new _AsyncDispatcher_654();
MockRM rm1 = new _MockRM_677(dispatcher, conf);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
rm1.KillApp(app1.GetApplicationId());
FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
.Succeeded, string.Empty, string.Empty);
am1.UnregisterAppAttempt(req, true);
rm1.WaitForState(am1.GetApplicationAttemptId(), RMAppAttemptState.Finishing);
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
rm1.WaitForState(am1.GetApplicationAttemptId(), RMAppAttemptState.Finished);
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Finished);
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerCleanupWhenRMRestartedAppNotRegistered()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// start RM
DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm1 = new _MockRM_413(dispatcher, conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// create app and launch the AM
RMApp app0 = rm1.SubmitApp(200);
MockAM am0 = LaunchAM(app0, rm1, nm1);
nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Running);
rm1.WaitForState(app0.GetApplicationId(), RMAppState.Running);
// start new RM
DrainDispatcher dispatcher2 = new DrainDispatcher();
MockRM rm2 = new _MockRM_432(dispatcher2, conf, memStore);
rm2.Start();
// nm1 register to rm2, and do a heartbeat
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
rm2.WaitForState(app0.GetApplicationId(), RMAppState.Accepted);
// Add unknown container for application unknown to scheduler
NodeHeartbeatResponse response = nm1.NodeHeartbeat(am0.GetApplicationAttemptId(),
2, ContainerState.Running);
WaitForContainerCleanup(dispatcher2, nm1, response);
rm1.Stop();
rm2.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestRMIdentifierOnContainerAllocation()
{
MockRM rm = new MockRM(conf);
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(2048);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler
nm1.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(1000);
alloc1Response = am1.Schedule();
}
// assert RMIdentifer is set properly in allocated containers
Container allocatedContainer = alloc1Response.GetAllocatedContainers()[0];
ContainerTokenIdentifier tokenId = BuilderUtils.NewContainerTokenIdentifier(allocatedContainer
.GetContainerToken());
NUnit.Framework.Assert.AreEqual(MockRM.GetClusterTimeStamp(), tokenId.GetRMIdentifier
());
rm.Stop();
}
public virtual void TestAllocateContainerOnNodeWithoutOffSwitchSpecified()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
RMApp app1 = rm.SubmitApp(2048);
// kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
// add request for containers
IList <ResourceRequest> requests = new AList <ResourceRequest>();
requests.AddItem(am1.CreateResourceReq("127.0.0.1", 1 * Gb, 1, 1));
requests.AddItem(am1.CreateResourceReq("/default-rack", 1 * Gb, 1, 1));
am1.Allocate(requests, null);
// send the request
try
{
// kick the schedule
nm1.NodeHeartbeat(true);
}
catch (ArgumentNullException)
{
NUnit.Framework.Assert.Fail("NPE when allocating container on node but " + "forget to set off-switch request should be handled"
);
}
rm.Stop();
}