/// <exception cref="System.Exception"/>
public virtual void TestAppOnMultiNode()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
conf.Set("yarn.scheduler.capacity.node-locality-delay", "-1");
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
MockNM nm2 = rm.RegisterNode("h2:5678", 10240);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
//request for containers
int request = 13;
am.Allocate("h1", 1000, request, new AList <ContainerId>());
//kick the scheduler
IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
>()).GetAllocatedContainers();
int contReceived = conts.Count;
while (contReceived < 3)
{
//only 3 containers are available on node1
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 3);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(3, conts.Count);
//send node2 heartbeat
conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
contReceived = conts.Count;
while (contReceived < 10)
{
nm2.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 10);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(10, conts.Count);
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidContainerReleaseRequest()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(1024);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler
nm1.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(1000);
alloc1Response = am1.Schedule();
}
NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0);
RMApp app2 = rm.SubmitApp(1024);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId());
am2.RegisterAppAttempt();
// Now trying to release container allocated for app1 -> appAttempt1.
ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId();
am2.AddContainerToBeReleased(cId);
try
{
am2.Schedule();
NUnit.Framework.Assert.Fail("Exception was expected!!");
}
catch (InvalidContainerReleaseException e)
{
StringBuilder sb = new StringBuilder("Cannot release container : ");
sb.Append(cId.ToString());
sb.Append(" not belonging to this application attempt : ");
sb.Append(attempt2.GetAppAttemptId().ToString());
NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString()));
}
}
finally
{
if (rm != null)
{
rm.Stop();
}
}
}
public virtual void TestAMLaunchAndCleanup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
TestApplicationMasterLauncher.MyContainerManagerImpl containerManager = new TestApplicationMasterLauncher.MyContainerManagerImpl
();
MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
// kick the scheduling
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.launched == false && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue(containerManager.launched);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId appAttemptId = attempt.GetAppAttemptId();
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), containerManager.attemptIdAtContainerManager
);
NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), containerManager.submitTimeAtContainerManager
);
NUnit.Framework.Assert.AreEqual(app.GetRMAppAttempt(appAttemptId).GetMasterContainer
().GetId().ToString(), containerManager.containerIdAtContainerManager);
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId().ToString(), containerManager.nmHostAtContainerManager
);
NUnit.Framework.Assert.AreEqual(YarnConfiguration.DefaultRmAmMaxAttempts, containerManager
.maxAppAttempts);
MockAM am = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), appAttemptId
);
am.RegisterAppAttempt();
am.UnregisterAppAttempt();
//complete the AM container to finish the app normally
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
waitCount = 0;
while (containerManager.cleanedup == false && waitCount++ < 20)
{
Log.Info("Waiting for AM Cleanup to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue(containerManager.cleanedup);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
// Test even if AM container is allocated with containerId not equal to 1, the
// following allocate requests from AM should be able to retrieve the
// corresponding NM Token.
/// <exception cref="System.Exception"/>
public virtual void TestNMTokenSentForNormalContainer()
{
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName
());
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
// Call getNewContainerId to increase container Id so that the AM container
// Id doesn't equal to one.
CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId();
// kick the scheduling
nm1.NodeHeartbeat(true);
MockAM am = MockRM.LaunchAM(app, rm, nm1);
// am container Id not equal to 1.
NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId
() != 1);
// NMSecretManager doesn't record the node on which the am is allocated.
NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
int NumContainers = 1;
IList <Container> containers = new AList <Container>();
// nmTokens keeps track of all the nmTokens issued in the allocate call.
IList <NMToken> expectedNMTokens = new AList <NMToken>();
// am1 allocate 1 container on nm1.
while (true)
{
AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList
<ContainerId>());
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
if (containers.Count == NumContainers)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
NodeId nodeId = expectedNMTokens[0].GetNodeId();
// NMToken is sent for the allocated container.
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId);
}
/// <exception cref="System.Exception"/>
public static MockAM LaunchAM(RMApp app, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
rm, MockNM nm)
{
rm.WaitForState(app.GetApplicationId(), RMAppState.Accepted);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
System.Console.Out.WriteLine("Launch AM " + attempt.GetAppAttemptId());
nm.NodeHeartbeat(true);
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
rm.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Launched);
return(am);
}
/// <exception cref="System.Exception"/>
private void TestMinimumAllocation(YarnConfiguration conf, int testAlloc)
{
MockRM rm = new MockRM(conf);
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(testAlloc);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId
());
int checkAlloc = conf.GetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, YarnConfiguration
.DefaultRmSchedulerMinimumAllocationMb);
NUnit.Framework.Assert.AreEqual(checkAlloc, report_nm1.GetUsedResource().GetMemory
());
rm.Stop();
}
// This is to test fetching AM container will be retried, if AM container is
// not fetchable since DNS is unavailable causing container token/NMtoken
// creation failure.
/// <exception cref="System.Exception"/>
public virtual void TestAMContainerAllocationWhenDNSUnavailable()
{
MockRM rm1 = new _MockRM_303(this, conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
SecurityUtilTestHelper.SetTokenServiceUseIp(true);
RMApp app1 = rm1.SubmitApp(200);
RMAppAttempt attempt = app1.GetCurrentAppAttempt();
nm1.NodeHeartbeat(true);
// fetching am container will fail, keep retrying 5 times.
while (numRetries <= 5)
{
nm1.NodeHeartbeat(true);
Sharpen.Thread.Sleep(1000);
NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Scheduled, attempt.GetAppAttemptState
());
System.Console.Out.WriteLine("Waiting for am container to be allocated.");
}
SecurityUtilTestHelper.SetTokenServiceUseIp(false);
rm1.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Allocated);
MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
}
private static RMApp CreateRMApp(ApplicationId appId)
{
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L);
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
));
RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId
.NewInstance(appId, 1));
Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt);
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null,
0, 0, int.MaxValue, long.MaxValue));
return(app);
}
public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, string user, string
schemePrefix)
{
this.startTime = 0;
this.containerId = string.Empty;
this.nodeHttpAddress = string.Empty;
this.nodeId = string.Empty;
this.logsLink = string.Empty;
this.blacklistedNodes = string.Empty;
if (attempt != null)
{
this.id = attempt.GetAppAttemptId().GetAttemptId();
this.startTime = attempt.GetStartTime();
Container masterContainer = attempt.GetMasterContainer();
if (masterContainer != null)
{
this.containerId = masterContainer.GetId().ToString();
this.nodeHttpAddress = masterContainer.GetNodeHttpAddress();
this.nodeId = masterContainer.GetNodeId().ToString();
this.logsLink = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer.GetNodeHttpAddress
(), ConverterUtils.ToString(masterContainer.GetId()), user);
if (rm.GetResourceScheduler() is AbstractYarnScheduler)
{
AbstractYarnScheduler ayScheduler = (AbstractYarnScheduler)rm.GetResourceScheduler
();
SchedulerApplicationAttempt sattempt = ayScheduler.GetApplicationAttempt(attempt.
GetAppAttemptId());
if (sattempt != null)
{
blacklistedNodes = StringUtils.Join(sattempt.GetBlacklistedNodes(), ", ");
}
}
}
}
}
// Test regular RM restart/failover, new RM should not count
// AM failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestRMRestartOrFailoverNotCountedForAMFailures()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
// AM should be restarted even though max-am-attempt is 1.
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt1).MayBeLastAttempt());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
// re-register the NM
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus
>();
status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager);
status.SetContainerId(attempt1.GetMasterContainer().GetId());
status.SetContainerState(ContainerState.Complete);
status.SetDiagnostics(string.Empty);
nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null);
rm2.WaitForState(attempt1.GetAppAttemptId(), RMAppAttemptState.Failed);
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledByResourcemanager, appState
.GetAttempt(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Will automatically start a new AppAttempt in rm2
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt3 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt3.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
/// <exception cref="System.Exception"/>
protected internal virtual ContainerId StoreAttempt(RMStateStore store, ApplicationAttemptId
attemptId, string containerIdStr, Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier
> appToken, SecretKey clientTokenMasterKey, RMStateStoreTestBase.TestDispatcher
dispatcher)
{
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Container container = new ContainerPBImpl();
container.SetId(ConverterUtils.ToContainerId(containerIdStr));
RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
Org.Mockito.Mockito.When(mockAttempt.GetAMRMToken()).ThenReturn(appToken);
Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
);
Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
dispatcher.attemptId = attemptId;
store.StoreNewApplicationAttempt(mockAttempt);
WaitNotify(dispatcher);
return(container.GetId());
}
protected override void GenerateApplicationTable(HtmlBlock.Block html, UserGroupInformation
callerUGI, ICollection <ApplicationAttemptReport> attempts)
{
// Application Attempt Table
Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html
.Table("#attempts").Thead().Tr().Th(".id", "Attempt ID").Th(".started", "Started"
).Th(".node", "Node").Th(".logs", "Logs").Th(".blacklistednodes", "Blacklisted Nodes"
).().().Tbody();
RMApp rmApp = this.rm.GetRMContext().GetRMApps()[this.appID];
if (rmApp == null)
{
return;
}
StringBuilder attemptsTableData = new StringBuilder("[\n");
foreach (ApplicationAttemptReport appAttemptReport in attempts)
{
RMAppAttempt rmAppAttempt = rmApp.GetRMAppAttempt(appAttemptReport.GetApplicationAttemptId
());
if (rmAppAttempt == null)
{
continue;
}
AppAttemptInfo attemptInfo = new AppAttemptInfo(this.rm, rmAppAttempt, rmApp.GetUser
(), WebAppUtils.GetHttpSchemePrefix(conf));
string blacklistedNodesCount = "N/A";
ICollection <string> nodes = RMAppAttemptBlock.GetBlacklistedNodes(rm, rmAppAttempt
.GetAppAttemptId());
if (nodes != null)
{
blacklistedNodesCount = nodes.Count.ToString();
}
string nodeLink = attemptInfo.GetNodeHttpAddress();
if (nodeLink != null)
{
nodeLink = WebAppUtils.GetHttpSchemePrefix(conf) + nodeLink;
}
string logsLink = attemptInfo.GetLogsLink();
attemptsTableData.Append("[\"<a href='").Append(Url("appattempt", rmAppAttempt.GetAppAttemptId
().ToString())).Append("'>").Append(rmAppAttempt.GetAppAttemptId().ToString()).Append
("</a>\",\"").Append(attemptInfo.GetStartTime()).Append("\",\"<a ").Append(nodeLink
== null ? "#" : "href='" + nodeLink).Append("'>").Append(nodeLink == null ? "N/A"
: StringEscapeUtils.EscapeJavaScript(StringEscapeUtils.EscapeHtml(nodeLink))).Append
("</a>\",\"<a ").Append(logsLink == null ? "#" : "href='" + logsLink).Append("'>"
).Append(logsLink == null ? "N/A" : "Logs").Append("</a>\",").Append("\"").Append
(blacklistedNodesCount).Append("\"],\n");
}
if (attemptsTableData[attemptsTableData.Length - 2] == ',')
{
attemptsTableData.Delete(attemptsTableData.Length - 2, attemptsTableData.Length -
1);
}
attemptsTableData.Append("]");
html.Script().$type("text/javascript").("var attemptsTableData=" + attemptsTableData
).();
tbody.().();
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
private void Launch()
{
Connect();
ContainerId masterContainerID = masterContainer.GetId();
ApplicationSubmissionContext applicationContext = application.GetSubmissionContext
();
Log.Info("Setting up container " + masterContainer + " for AM " + application.GetAppAttemptId
());
ContainerLaunchContext launchContext = CreateAMContainerLaunchContext(applicationContext
, masterContainerID);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext
, masterContainer.GetContainerToken());
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
StartContainersResponse response = containerMgrProxy.StartContainers(allRequests);
if (response.GetFailedRequests() != null && response.GetFailedRequests().Contains
(masterContainerID))
{
Exception t = response.GetFailedRequests()[masterContainerID].DeSerialize();
ParseAndThrowException(t);
}
else
{
Log.Info("Done launching container " + masterContainer + " for AM " + application
.GetAppAttemptId());
}
}
public virtual IDictionary <ApplicationAttemptId, RMAppAttempt> GetAppAttempts()
{
IDictionary <ApplicationAttemptId, RMAppAttempt> attempts = new LinkedHashMap <ApplicationAttemptId
, RMAppAttempt>();
if (attempt != null)
{
attempts[attempt.GetAppAttemptId()] = attempt;
}
return(attempts);
}
public virtual void AppAttemptRegistered(RMAppAttempt appAttempt, long registeredTime
)
{
if (publishSystemMetrics)
{
dispatcher.GetEventHandler().Handle(new AppAttemptRegisteredEvent(appAttempt.GetAppAttemptId
(), appAttempt.GetHost(), appAttempt.GetRpcPort(), appAttempt.GetTrackingUrl(),
appAttempt.GetOriginalTrackingUrl(), appAttempt.GetMasterContainer().GetId(), registeredTime
));
}
}
/// <exception cref="System.Exception"/>
private MockAM LaunchAM(RMApp app, MockRM rm, MockNM nm)
{
RMAppAttempt attempt = app.GetCurrentAppAttempt();
nm.NodeHeartbeat(true);
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
return(am);
}
public virtual void TestValidateResourceBlacklistRequest()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new
YarnConfiguration(), containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = "*";
RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
WaitForLaunchedState(attempt);
// Create a client to the RM.
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc
, rmBindAddress, conf));
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
client.RegisterApplicationMaster(request);
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
Sharpen.Collections.SingletonList(ResourceRequest.Any), null);
AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null
, blacklistRequest);
bool error = false;
try
{
client.Allocate(allocateRequest);
}
catch (InvalidResourceBlacklistRequestException)
{
error = true;
}
rm.Stop();
NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException"
, error);
}
public virtual void AppAttemptFinished(RMAppAttempt appAttempt, RMAppAttemptState
appAttemtpState, RMApp app, long finishedTime)
{
if (publishSystemMetrics)
{
dispatcher.GetEventHandler().Handle(new AppAttemptFinishedEvent(appAttempt.GetAppAttemptId
(), appAttempt.GetTrackingUrl(), appAttempt.GetOriginalTrackingUrl(), appAttempt
.GetDiagnostics(), app.GetFinalApplicationStatus(), RMServerUtils.CreateApplicationAttemptState
(appAttemtpState), finishedTime));
}
}
public virtual void TestAuthorizedAccess()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = "*";
RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.containerTokens == null && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
WaitForLaunchedState(attempt);
// Create a client to the RM.
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this
, rpc, conf));
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request
);
NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
if (UserGroupInformation.IsSecurityEnabled())
{
NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
()).Length > 0);
}
NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs
()[ApplicationAccessType.ViewApp]);
}
/// <exception cref="System.Exception"/>
public virtual void TestAppWithNoContainers()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
/// <exception cref="System.Exception"/>
private void TestRMWritingMassiveHistory(MockRM rm)
{
rm.Start();
MockNM nm = rm.RegisterNode("127.0.0.1:1234", 1024 * 10100);
RMApp app = rm.SubmitApp(1024);
nm.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
int request = 10000;
am.Allocate("127.0.0.1", 1024, request, new AList <ContainerId>());
nm.NodeHeartbeat(true);
IList <Container> allocated = am.Allocate(new AList <ResourceRequest>(), new AList <
ContainerId>()).GetAllocatedContainers();
int waitCount = 0;
int allocatedSize = allocated.Count;
while (allocatedSize < request && waitCount++ < 200)
{
Sharpen.Thread.Sleep(300);
allocated = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
allocatedSize += allocated.Count;
nm.NodeHeartbeat(true);
}
NUnit.Framework.Assert.AreEqual(request, allocatedSize);
am.UnregisterAppAttempt();
am.WaitForState(RMAppAttemptState.Finishing);
nm.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
NodeHeartbeatResponse resp = nm.NodeHeartbeat(true);
IList <ContainerId> cleaned = resp.GetContainersToCleanup();
int cleanedSize = cleaned.Count;
waitCount = 0;
while (cleanedSize < allocatedSize && waitCount++ < 200)
{
Sharpen.Thread.Sleep(300);
resp = nm.NodeHeartbeat(true);
cleaned = resp.GetContainersToCleanup();
cleanedSize += cleaned.Count;
}
NUnit.Framework.Assert.AreEqual(allocatedSize, cleanedSize);
rm.WaitForState(app.GetApplicationId(), RMAppState.Finished);
rm.Stop();
}
public virtual void TestARRMResponseId()
{
MockNM nm1 = rm.RegisterNode("h1:1234", 5000);
RMApp app = rm.SubmitApp(2000);
// Trigger the scheduling so the AM gets 'launched'
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0F, null, null,
null);
AllocateResponse response = Allocate(attempt.GetAppAttemptId(), allocateRequest);
NUnit.Framework.Assert.AreEqual(1, response.GetResponseId());
NUnit.Framework.Assert.IsTrue(response.GetAMCommand() == null);
allocateRequest = AllocateRequest.NewInstance(response.GetResponseId(), 0F, null,
null, null);
response = Allocate(attempt.GetAppAttemptId(), allocateRequest);
NUnit.Framework.Assert.AreEqual(2, response.GetResponseId());
/* try resending */
response = Allocate(attempt.GetAppAttemptId(), allocateRequest);
NUnit.Framework.Assert.AreEqual(2, response.GetResponseId());
allocateRequest = AllocateRequest.NewInstance(0, 0F, null, null, null);
try
{
Allocate(attempt.GetAppAttemptId(), allocateRequest);
NUnit.Framework.Assert.Fail();
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue(e.InnerException is InvalidApplicationMasterRequestException
);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestUsageWithOneAttemptAndOneContainer()
{
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());
nm.RegisterNode();
RMApp app0 = rm.SubmitApp(200);
RMAppMetrics rmAppMetrics = app0.GetRMAppMetrics();
NUnit.Framework.Assert.IsTrue("Before app submittion, memory seconds should have been 0 but was "
+ rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() == 0);
NUnit.Framework.Assert.IsTrue("Before app submission, vcore seconds should have been 0 but was "
+ rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() == 0);
RMAppAttempt attempt0 = app0.GetCurrentAppAttempt();
nm.NodeHeartbeat(true);
MockAM am0 = rm.SendAMLaunched(attempt0.GetAppAttemptId());
am0.RegisterAppAttempt();
RMContainer rmContainer = rm.GetResourceScheduler().GetRMContainer(attempt0.GetMasterContainer
().GetId());
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
while (rmAppMetrics.GetMemorySeconds() <= 0 && cumulativeSleepTime < 5000)
{
Sharpen.Thread.Sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
}
rmAppMetrics = app0.GetRMAppMetrics();
NUnit.Framework.Assert.IsTrue("While app is running, memory seconds should be >0 but is "
+ rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() > 0);
NUnit.Framework.Assert.IsTrue("While app is running, vcore seconds should be >0 but is "
+ rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() > 0);
MockRM.FinishAMAndVerifyAppState(app0, rm, nm, am0);
AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(rmContainer);
rmAppMetrics = app0.GetRMAppMetrics();
NUnit.Framework.Assert.AreEqual("Unexcpected MemorySeconds value", ru.GetMemorySeconds
(), rmAppMetrics.GetMemorySeconds());
NUnit.Framework.Assert.AreEqual("Unexpected VcoreSeconds value", ru.GetVcoreSeconds
(), rmAppMetrics.GetVcoreSeconds());
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestResourceTypes()
{
Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes> >
driver = new Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
> >();
CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();
csconf.SetResourceComparator(typeof(DominantResourceCalculator));
YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf);
testCapacityDRConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
), typeof(ResourceScheduler));
YarnConfiguration testCapacityDefConf = new YarnConfiguration();
testCapacityDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
), typeof(ResourceScheduler));
YarnConfiguration testFairDefConf = new YarnConfiguration();
testFairDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(FairScheduler), typeof(
ResourceScheduler));
driver[conf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory);
driver[testCapacityDRConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.
Cpu, YarnServiceProtos.SchedulerResourceTypes.Memory);
driver[testCapacityDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes
.Memory);
driver[testFairDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory
, YarnServiceProtos.SchedulerResourceTypes.Cpu);
foreach (KeyValuePair <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
> > entry in driver)
{
EnumSet <YarnServiceProtos.SchedulerResourceTypes> expectedValue = entry.Value;
MockRM rm = new MockRM(entry.Key);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
RMApp app1 = rm.SubmitApp(2048);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
RegisterApplicationMasterResponse resp = am1.RegisterAppAttempt();
EnumSet <YarnServiceProtos.SchedulerResourceTypes> types = resp.GetSchedulerResourceTypes
();
Log.Info("types = " + types.ToString());
NUnit.Framework.Assert.AreEqual(expectedValue, types);
rm.Stop();
}
}
private static RMAppAttempt CreateRMAppAttempt(ApplicationAttemptId appAttemptId)
{
RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(appAttemptId);
Org.Mockito.Mockito.When(appAttempt.GetHost()).ThenReturn("test host");
Org.Mockito.Mockito.When(appAttempt.GetRpcPort()).ThenReturn(-100);
Container container = Org.Mockito.Mockito.Mock <Container>();
Org.Mockito.Mockito.When(container.GetId()).ThenReturn(ContainerId.NewContainerId
(appAttemptId, 1));
Org.Mockito.Mockito.When(appAttempt.GetMasterContainer()).ThenReturn(container);
Org.Mockito.Mockito.When(appAttempt.GetDiagnostics()).ThenReturn("test diagnostics info"
);
Org.Mockito.Mockito.When(appAttempt.GetTrackingUrl()).ThenReturn("test url");
Org.Mockito.Mockito.When(appAttempt.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
return(appAttempt);
}
public virtual AggregateAppResourceUsage GetAggregateAppResourceUsage()
{
long memorySeconds = finishedMemorySeconds.Get();
long vcoreSeconds = finishedVcoreSeconds.Get();
// Only add in the running containers if this is the active attempt.
RMAppAttempt currentAttempt = rmContext.GetRMApps()[attemptId.GetApplicationId()]
.GetCurrentAppAttempt();
if (currentAttempt.GetAppAttemptId().Equals(attemptId))
{
ApplicationResourceUsageReport appResUsageReport = rmContext.GetScheduler().GetAppResourceUsageReport
(attemptId);
if (appResUsageReport != null)
{
memorySeconds += appResUsageReport.GetMemorySeconds();
vcoreSeconds += appResUsageReport.GetVcoreSeconds();
}
}
return(new AggregateAppResourceUsage(memorySeconds, vcoreSeconds));
}
/// <exception cref="System.Exception"/>
private void CheckActiveRMFunctionality()
{
NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Active
, rm.adminService.GetServiceStatus().GetState());
NUnit.Framework.Assert.IsTrue("Active RM services aren't started", rm.AreActiveServicesRunning
());
NUnit.Framework.Assert.IsTrue("RM is not ready to become active", rm.adminService
.GetServiceStatus().IsReadyToBecomeActive());
try
{
rm.GetNewAppId();
rm.RegisterNode("127.0.0.1:1", 2048);
app = rm.SubmitApp(1024);
attempt = app.GetCurrentAppAttempt();
rm.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Scheduled);
}
catch (Exception e)
{
NUnit.Framework.Assert.Fail("Unable to perform Active RM functions");
Log.Error("ActiveRM check failed", e);
}
CheckActiveRMWebServices();
}
/// <exception cref="System.Exception"/>
public virtual void TestRMIdentifierOnContainerAllocation()
{
MockRM rm = new MockRM(conf);
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(2048);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler
nm1.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(1000);
alloc1Response = am1.Schedule();
}
// assert RMIdentifer is set properly in allocated containers
Container allocatedContainer = alloc1Response.GetAllocatedContainers()[0];
ContainerTokenIdentifier tokenId = BuilderUtils.NewContainerTokenIdentifier(allocatedContainer
.GetContainerToken());
NUnit.Framework.Assert.AreEqual(MockRM.GetClusterTimeStamp(), tokenId.GetRMIdentifier
());
rm.Stop();
}
public virtual void TestAllocateContainerOnNodeWithoutOffSwitchSpecified()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
RMApp app1 = rm.SubmitApp(2048);
// kick the scheduling, 2 GB given to AM1, remaining 4GB on nm1
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
// add request for containers
IList <ResourceRequest> requests = new AList <ResourceRequest>();
requests.AddItem(am1.CreateResourceReq("127.0.0.1", 1 * Gb, 1, 1));
requests.AddItem(am1.CreateResourceReq("/default-rack", 1 * Gb, 1, 1));
am1.Allocate(requests, null);
// send the request
try
{
// kick the schedule
nm1.NodeHeartbeat(true);
}
catch (ArgumentNullException)
{
NUnit.Framework.Assert.Fail("NPE when allocating container on node but " + "forget to set off-switch request should be handled"
);
}
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestActivatingApplicationAfterAddingNM()
{
MockRM rm1 = new MockRM(conf);
// start like normal because state is empty
rm1.Start();
// app that gets launched
RMApp app1 = rm1.SubmitApp(200);
// app that does not get launched
RMApp app2 = rm1.SubmitApp(200);
// app1 and app2 should be scheduled, but because no resource is available,
// they are not activated.
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
ApplicationAttemptId attemptId1 = attempt1.GetAppAttemptId();
rm1.WaitForState(attemptId1, RMAppAttemptState.Scheduled);
RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
ApplicationAttemptId attemptId2 = attempt2.GetAppAttemptId();
rm1.WaitForState(attemptId2, RMAppAttemptState.Scheduled);
MockNM nm1 = new MockNM("h1:1234", 15120, rm1.GetResourceTrackerService());
MockNM nm2 = new MockNM("h2:5678", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
nm2.RegisterNode();
//kick the scheduling
nm1.NodeHeartbeat(true);
// app1 should be allocated now
rm1.WaitForState(attemptId1, RMAppAttemptState.Allocated);
rm1.WaitForState(attemptId2, RMAppAttemptState.Scheduled);
nm2.NodeHeartbeat(true);
// app2 should be allocated now
rm1.WaitForState(attemptId1, RMAppAttemptState.Allocated);
rm1.WaitForState(attemptId2, RMAppAttemptState.Allocated);
rm1.Stop();
}