protected internal virtual Runnable CreateRunnableLauncher(RMAppAttempt application
, AMLauncherEventType @event)
{
Runnable launcher = new AMLauncher(context, application, @event, GetConfig());
return(launcher);
}
// This is to test fetching AM container will be retried, if AM container is
// not fetchable since DNS is unavailable causing container token/NMtoken
// creation failure.
/// <exception cref="System.Exception"/>
public virtual void TestAMContainerAllocationWhenDNSUnavailable()
{
MockRM rm1 = new _MockRM_303(this, conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
SecurityUtilTestHelper.SetTokenServiceUseIp(true);
RMApp app1 = rm1.SubmitApp(200);
RMAppAttempt attempt = app1.GetCurrentAppAttempt();
nm1.NodeHeartbeat(true);
// fetching am container will fail, keep retrying 5 times.
while (numRetries <= 5)
{
nm1.NodeHeartbeat(true);
Sharpen.Thread.Sleep(1000);
NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Scheduled, attempt.GetAppAttemptState
());
System.Console.Out.WriteLine("Waiting for am container to be allocated.");
}
SecurityUtilTestHelper.SetTokenServiceUseIp(false);
rm1.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Allocated);
MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
}
/// <exception cref="System.Exception"/>
private void TestMinimumAllocation(YarnConfiguration conf, int testAlloc)
{
MockRM rm = new MockRM(conf);
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(testAlloc);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId
());
int checkAlloc = conf.GetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, YarnConfiguration
.DefaultRmSchedulerMinimumAllocationMb);
NUnit.Framework.Assert.AreEqual(checkAlloc, report_nm1.GetUsedResource().GetMemory
());
rm.Stop();
}
private static RMApp CreateRMApp(ApplicationId appId)
{
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L);
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
));
RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId
.NewInstance(appId, 1));
Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt);
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null,
0, 0, int.MaxValue, long.MaxValue));
return(app);
}
public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, string user, string
schemePrefix)
{
this.startTime = 0;
this.containerId = string.Empty;
this.nodeHttpAddress = string.Empty;
this.nodeId = string.Empty;
this.logsLink = string.Empty;
this.blacklistedNodes = string.Empty;
if (attempt != null)
{
this.id = attempt.GetAppAttemptId().GetAttemptId();
this.startTime = attempt.GetStartTime();
Container masterContainer = attempt.GetMasterContainer();
if (masterContainer != null)
{
this.containerId = masterContainer.GetId().ToString();
this.nodeHttpAddress = masterContainer.GetNodeHttpAddress();
this.nodeId = masterContainer.GetNodeId().ToString();
this.logsLink = WebAppUtils.GetRunningLogURL(schemePrefix + masterContainer.GetNodeHttpAddress
(), ConverterUtils.ToString(masterContainer.GetId()), user);
if (rm.GetResourceScheduler() is AbstractYarnScheduler)
{
AbstractYarnScheduler ayScheduler = (AbstractYarnScheduler)rm.GetResourceScheduler
();
SchedulerApplicationAttempt sattempt = ayScheduler.GetApplicationAttempt(attempt.
GetAppAttemptId());
if (sattempt != null)
{
blacklistedNodes = StringUtils.Join(sattempt.GetBlacklistedNodes(), ", ");
}
}
}
}
}
/// <exception cref="System.Exception"/>
protected internal virtual ContainerId StoreAttempt(RMStateStore store, ApplicationAttemptId
attemptId, string containerIdStr, Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier
> appToken, SecretKey clientTokenMasterKey, RMStateStoreTestBase.TestDispatcher
dispatcher)
{
RMAppAttemptMetrics mockRmAppAttemptMetrics = Org.Mockito.Mockito.Mock <RMAppAttemptMetrics
>();
Container container = new ContainerPBImpl();
container.SetId(ConverterUtils.ToContainerId(containerIdStr));
RMAppAttempt mockAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(mockAttempt.GetAppAttemptId()).ThenReturn(attemptId);
Org.Mockito.Mockito.When(mockAttempt.GetMasterContainer()).ThenReturn(container);
Org.Mockito.Mockito.When(mockAttempt.GetAMRMToken()).ThenReturn(appToken);
Org.Mockito.Mockito.When(mockAttempt.GetClientTokenMasterKey()).ThenReturn(clientTokenMasterKey
);
Org.Mockito.Mockito.When(mockAttempt.GetRMAppAttemptMetrics()).ThenReturn(mockRmAppAttemptMetrics
);
Org.Mockito.Mockito.When(mockRmAppAttemptMetrics.GetAggregateAppResourceUsage()).
ThenReturn(new AggregateAppResourceUsage(0, 0));
dispatcher.attemptId = attemptId;
store.StoreNewApplicationAttempt(mockAttempt);
WaitNotify(dispatcher);
return(container.GetId());
}
/// <summary>create a summary of the application's runtime.</summary>
/// <param name="app">
///
/// <see cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Rmapp.RMApp"/>
/// whose summary is to be created, cannot
/// be <code>null</code>.
/// </param>
public static RMAppManager.ApplicationSummary.SummaryBuilder CreateAppSummary(RMApp
app)
{
string trackingUrl = "N/A";
string host = "N/A";
RMAppAttempt attempt = app.GetCurrentAppAttempt();
if (attempt != null)
{
trackingUrl = attempt.GetTrackingUrl();
host = attempt.GetHost();
}
RMAppMetrics metrics = app.GetRMAppMetrics();
RMAppManager.ApplicationSummary.SummaryBuilder summary = new RMAppManager.ApplicationSummary.SummaryBuilder
().Add("appId", app.GetApplicationId()).Add("name", app.GetName()).Add("user", app
.GetUser()).Add("queue", app.GetQueue()).Add("state", app.GetState()).Add("trackingUrl"
, trackingUrl).Add("appMasterHost", host).Add("startTime", app.GetStartTime()).Add
("finishTime", app.GetFinishTime()).Add("finalStatus", app.GetFinalApplicationStatus
()).Add("memorySeconds", metrics.GetMemorySeconds()).Add("vcoreSeconds", metrics
.GetVcoreSeconds()).Add("preemptedAMContainers", metrics.GetNumAMContainersPreempted
()).Add("preemptedNonAMContainers", metrics.GetNumNonAMContainersPreempted()).Add
("preemptedResources", metrics.GetResourcePreempted()).Add("applicationType", app
.GetApplicationType());
return(summary);
}
// Test regular RM restart/failover, new RM should not count
// AM failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestRMRestartOrFailoverNotCountedForAMFailures()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
// AM should be restarted even though max-am-attempt is 1.
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt1).MayBeLastAttempt());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
// re-register the NM
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus
>();
status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager);
status.SetContainerId(attempt1.GetMasterContainer().GetId());
status.SetContainerState(ContainerState.Complete);
status.SetDiagnostics(string.Empty);
nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null);
rm2.WaitForState(attempt1.GetAppAttemptId(), RMAppAttemptState.Failed);
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledByResourcemanager, appState
.GetAttempt(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Will automatically start a new AppAttempt in rm2
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt3 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt3.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
private void Launch(RMAppAttempt application)
{
Runnable launcher = CreateRunnableLauncher(application, AMLauncherEventType.Launch
);
masterEvents.AddItem(launcher);
}
protected internal virtual ApplicationAttemptId CreateSchedulingRequest(int memory
, int vcores, string queueId, string userId, int numContainers, int priority)
{
ApplicationAttemptId id = CreateAppAttemptId(this.AppId++, this.AttemptId++);
scheduler.AddApplication(id.GetApplicationId(), queueId, userId, false);
// This conditional is for testAclSubmitApplication where app is rejected
// and no app is added.
if (scheduler.GetSchedulerApplications().Contains(id.GetApplicationId()))
{
scheduler.AddApplicationAttempt(id, false, false);
}
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest request = CreateResourceRequest(memory, vcores, ResourceRequest.Any
, priority, numContainers, true);
ask.AddItem(request);
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
RMAppAttempt rmAppAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(rmApp.GetCurrentAppAttempt()).ThenReturn(rmAppAttempt);
Org.Mockito.Mockito.When(rmAppAttempt.GetRMAppAttemptMetrics()).ThenReturn(new RMAppAttemptMetrics
(id, resourceManager.GetRMContext()));
resourceManager.GetRMContext().GetRMApps()[id.GetApplicationId()] = rmApp;
scheduler.Allocate(id, ask, new AList <ContainerId>(), null, null);
return(id);
}
public virtual void Handle(AMLauncherEvent appEvent)
{
lock (this)
{
AMLauncherEventType @event = appEvent.GetType();
RMAppAttempt application = appEvent.GetAppAttempt();
switch (@event)
{
case AMLauncherEventType.Launch:
{
Launch(application);
break;
}
case AMLauncherEventType.Cleanup:
{
Cleanup(application);
break;
}
default:
{
break;
}
}
}
}
// Test RM restarts after AM container is preempted, new RM should not count
// AM preemption failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestPreemptedAMRestartOnRMRestart()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler();
ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 1);
// Forcibly preempt the am container;
scheduler.KillContainer(scheduler.GetRMContainer(amContainer));
am1.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry());
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// state store has 1 attempt stored.
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
NUnit.Framework.Assert.AreEqual(1, appState.GetAttemptCount());
// attempt stored has the preempted container exit status.
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Preempted, appState.GetAttempt
(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode();
rm2.Start();
// Restarted RM should re-launch the am.
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt2 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt2.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
protected override void GenerateApplicationTable(HtmlBlock.Block html, UserGroupInformation
callerUGI, ICollection <ApplicationAttemptReport> attempts)
{
// Application Attempt Table
Hamlet.TBODY <Hamlet.TABLE <Org.Apache.Hadoop.Yarn.Webapp.Hamlet.Hamlet> > tbody = html
.Table("#attempts").Thead().Tr().Th(".id", "Attempt ID").Th(".started", "Started"
).Th(".node", "Node").Th(".logs", "Logs").Th(".blacklistednodes", "Blacklisted Nodes"
).().().Tbody();
RMApp rmApp = this.rm.GetRMContext().GetRMApps()[this.appID];
if (rmApp == null)
{
return;
}
StringBuilder attemptsTableData = new StringBuilder("[\n");
foreach (ApplicationAttemptReport appAttemptReport in attempts)
{
RMAppAttempt rmAppAttempt = rmApp.GetRMAppAttempt(appAttemptReport.GetApplicationAttemptId
());
if (rmAppAttempt == null)
{
continue;
}
AppAttemptInfo attemptInfo = new AppAttemptInfo(this.rm, rmAppAttempt, rmApp.GetUser
(), WebAppUtils.GetHttpSchemePrefix(conf));
string blacklistedNodesCount = "N/A";
ICollection <string> nodes = RMAppAttemptBlock.GetBlacklistedNodes(rm, rmAppAttempt
.GetAppAttemptId());
if (nodes != null)
{
blacklistedNodesCount = nodes.Count.ToString();
}
string nodeLink = attemptInfo.GetNodeHttpAddress();
if (nodeLink != null)
{
nodeLink = WebAppUtils.GetHttpSchemePrefix(conf) + nodeLink;
}
string logsLink = attemptInfo.GetLogsLink();
attemptsTableData.Append("[\"<a href='").Append(Url("appattempt", rmAppAttempt.GetAppAttemptId
().ToString())).Append("'>").Append(rmAppAttempt.GetAppAttemptId().ToString()).Append
("</a>\",\"").Append(attemptInfo.GetStartTime()).Append("\",\"<a ").Append(nodeLink
== null ? "#" : "href='" + nodeLink).Append("'>").Append(nodeLink == null ? "N/A"
: StringEscapeUtils.EscapeJavaScript(StringEscapeUtils.EscapeHtml(nodeLink))).Append
("</a>\",\"<a ").Append(logsLink == null ? "#" : "href='" + logsLink).Append("'>"
).Append(logsLink == null ? "N/A" : "Logs").Append("</a>\",").Append("\"").Append
(blacklistedNodesCount).Append("\"],\n");
}
if (attemptsTableData[attemptsTableData.Length - 2] == ',')
{
attemptsTableData.Delete(attemptsTableData.Length - 2, attemptsTableData.Length -
1);
}
attemptsTableData.Append("]");
html.Script().$type("text/javascript").("var attemptsTableData=" + attemptsTableData
).();
tbody.().();
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidContainerReleaseRequest()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(1024);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler
nm1.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(1000);
alloc1Response = am1.Schedule();
}
NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0);
RMApp app2 = rm.SubmitApp(1024);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId());
am2.RegisterAppAttempt();
// Now trying to release container allocated for app1 -> appAttempt1.
ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId();
am2.AddContainerToBeReleased(cId);
try
{
am2.Schedule();
NUnit.Framework.Assert.Fail("Exception was expected!!");
}
catch (InvalidContainerReleaseException e)
{
StringBuilder sb = new StringBuilder("Cannot release container : ");
sb.Append(cId.ToString());
sb.Append(" not belonging to this application attempt : ");
sb.Append(attempt2.GetAppAttemptId().ToString());
NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString()));
}
}
finally
{
if (rm != null)
{
rm.Stop();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAppOnMultiNode()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
conf.Set("yarn.scheduler.capacity.node-locality-delay", "-1");
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
MockNM nm2 = rm.RegisterNode("h2:5678", 10240);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
//request for containers
int request = 13;
am.Allocate("h1", 1000, request, new AList <ContainerId>());
//kick the scheduler
IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
>()).GetAllocatedContainers();
int contReceived = conts.Count;
while (contReceived < 3)
{
//only 3 containers are available on node1
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 3);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(3, conts.Count);
//send node2 heartbeat
conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
contReceived = conts.Count;
while (contReceived < 10)
{
nm2.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 10);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(10, conts.Count);
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
public virtual void ApplicationAttemptStarted(RMAppAttempt appAttempt)
{
if (historyServiceEnabled)
{
dispatcher.GetEventHandler().Handle(new WritingApplicationAttemptStartEvent(appAttempt
.GetAppAttemptId(), ApplicationAttemptStartData.NewInstance(appAttempt.GetAppAttemptId
(), appAttempt.GetHost(), appAttempt.GetRpcPort(), appAttempt.GetMasterContainer
().GetId())));
}
}
public AMLauncher(RMContext rmContext, RMAppAttempt application, AMLauncherEventType
eventType, Configuration conf)
{
this.application = application;
this.conf = conf;
this.eventType = eventType;
this.rmContext = rmContext;
this.handler = rmContext.GetDispatcher().GetEventHandler();
this.masterContainer = application.GetMasterContainer();
}
/// <exception cref="System.Exception"/>
private void WaitForContainersToFinish(int expectedNum, RMAppAttempt attempt)
{
int count = 0;
while (attempt.GetJustFinishedContainers().Count < expectedNum && count < 500)
{
Sharpen.Thread.Sleep(100);
count++;
}
}
/// <exception cref="System.Exception"/>
private MockAM LaunchAM(RMApp app, MockRM rm, MockNM nm)
{
RMAppAttempt attempt = app.GetCurrentAppAttempt();
nm.NodeHeartbeat(true);
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
return(am);
}
public virtual void AppAttemptFinished(RMAppAttempt appAttempt, RMAppAttemptState
appAttemtpState, RMApp app, long finishedTime)
{
if (publishSystemMetrics)
{
dispatcher.GetEventHandler().Handle(new AppAttemptFinishedEvent(appAttempt.GetAppAttemptId
(), appAttempt.GetTrackingUrl(), appAttempt.GetOriginalTrackingUrl(), appAttempt
.GetDiagnostics(), app.GetFinalApplicationStatus(), RMServerUtils.CreateApplicationAttemptState
(appAttemtpState), finishedTime));
}
}
public virtual void AppAttemptRegistered(RMAppAttempt appAttempt, long registeredTime
)
{
if (publishSystemMetrics)
{
dispatcher.GetEventHandler().Handle(new AppAttemptRegisteredEvent(appAttempt.GetAppAttemptId
(), appAttempt.GetHost(), appAttempt.GetRpcPort(), appAttempt.GetTrackingUrl(),
appAttempt.GetOriginalTrackingUrl(), appAttempt.GetMasterContainer().GetId(), registeredTime
));
}
}
public virtual void TestValidateResourceBlacklistRequest()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
TestAMAuthorization.MockRMWithAMS rm = new TestAMAuthorization.MockRMWithAMS(new
YarnConfiguration(), containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = "*";
RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
WaitForLaunchedState(attempt);
// Create a client to the RM.
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_626(rpc
, rmBindAddress, conf));
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
client.RegisterApplicationMaster(request);
ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.NewInstance(
Sharpen.Collections.SingletonList(ResourceRequest.Any), null);
AllocateRequest allocateRequest = AllocateRequest.NewInstance(0, 0.0f, null, null
, blacklistRequest);
bool error = false;
try
{
client.Allocate(allocateRequest);
}
catch (InvalidResourceBlacklistRequestException)
{
error = true;
}
rm.Stop();
NUnit.Framework.Assert.IsTrue("Didn't not catch InvalidResourceBlacklistRequestException"
, error);
}
public virtual void TestAMLaunchAndCleanup()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
TestApplicationMasterLauncher.MyContainerManagerImpl containerManager = new TestApplicationMasterLauncher.MyContainerManagerImpl
();
MockRMWithCustomAMLauncher rm = new MockRMWithCustomAMLauncher(containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
// kick the scheduling
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.launched == false && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue(containerManager.launched);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId appAttemptId = attempt.GetAppAttemptId();
NUnit.Framework.Assert.AreEqual(appAttemptId.ToString(), containerManager.attemptIdAtContainerManager
);
NUnit.Framework.Assert.AreEqual(app.GetSubmitTime(), containerManager.submitTimeAtContainerManager
);
NUnit.Framework.Assert.AreEqual(app.GetRMAppAttempt(appAttemptId).GetMasterContainer
().GetId().ToString(), containerManager.containerIdAtContainerManager);
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId().ToString(), containerManager.nmHostAtContainerManager
);
NUnit.Framework.Assert.AreEqual(YarnConfiguration.DefaultRmAmMaxAttempts, containerManager
.maxAppAttempts);
MockAM am = new MockAM(rm.GetRMContext(), rm.GetApplicationMasterService(), appAttemptId
);
am.RegisterAppAttempt();
am.UnregisterAppAttempt();
//complete the AM container to finish the app normally
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
waitCount = 0;
while (containerManager.cleanedup == false && waitCount++ < 20)
{
Log.Info("Waiting for AM Cleanup to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsTrue(containerManager.cleanedup);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
private RMAppAttempt GetRMAppAttempt()
{
ApplicationId appId = this.appAttemptId.GetApplicationId();
RMAppAttempt attempt = null;
RMApp rmApp = rm.GetRMContext().GetRMApps()[appId];
if (rmApp != null)
{
attempt = rmApp.GetAppAttempts()[appAttemptId];
}
return(attempt);
}
public virtual void TestAuthorizedAccess()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = "*";
RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.containerTokens == null && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
WaitForLaunchedState(attempt);
// Create a client to the RM.
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this
, rpc, conf));
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request
);
NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
if (UserGroupInformation.IsSecurityEnabled())
{
NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
()).Length > 0);
}
NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs
()[ApplicationAccessType.ViewApp]);
}
public virtual void SetApplicationHeadroomForMetrics(Org.Apache.Hadoop.Yarn.Api.Records.Resource
headroom)
{
RMAppAttempt attempt = rmContext.GetRMApps()[attemptId.GetApplicationId()].GetCurrentAppAttempt
();
if (attempt != null)
{
attempt.GetRMAppAttemptMetrics().SetApplicationAttemptHeadRoom(Resources.Clone(headroom
));
}
}
// resourceLimit: updated when LeafQueue#recoverContainer#allocateResource
// is called.
// newlyAllocatedContainers.add(rmContainer);
// schedulingOpportunities
// lastScheduledContainer
public virtual void IncNumAllocatedContainers(NodeType containerType, NodeType requestType
)
{
RMAppAttempt attempt = rmContext.GetRMApps()[attemptId.GetApplicationId()].GetCurrentAppAttempt
();
if (attempt != null)
{
attempt.GetRMAppAttemptMetrics().IncNumAllocatedContainers(containerType, requestType
);
}
}
public virtual void ApplicationAttemptFinished(RMAppAttempt appAttempt, RMAppAttemptState
finalState)
{
if (historyServiceEnabled)
{
dispatcher.GetEventHandler().Handle(new WritingApplicationAttemptFinishEvent(appAttempt
.GetAppAttemptId(), ApplicationAttemptFinishData.NewInstance(appAttempt.GetAppAttemptId
(), appAttempt.GetDiagnostics().ToString(), appAttempt.GetTrackingUrl(), appAttempt
.GetFinalApplicationStatus(), RMServerUtils.CreateApplicationAttemptState(finalState
))));
}
}
// Test even if AM container is allocated with containerId not equal to 1, the
// following allocate requests from AM should be able to retrieve the
// corresponding NM Token.
/// <exception cref="System.Exception"/>
public virtual void TestNMTokenSentForNormalContainer()
{
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName
());
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
// Call getNewContainerId to increase container Id so that the AM container
// Id doesn't equal to one.
CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId();
// kick the scheduling
nm1.NodeHeartbeat(true);
MockAM am = MockRM.LaunchAM(app, rm, nm1);
// am container Id not equal to 1.
NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId
() != 1);
// NMSecretManager doesn't record the node on which the am is allocated.
NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
int NumContainers = 1;
IList <Container> containers = new AList <Container>();
// nmTokens keeps track of all the nmTokens issued in the allocate call.
IList <NMToken> expectedNMTokens = new AList <NMToken>();
// am1 allocate 1 container on nm1.
while (true)
{
AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList
<ContainerId>());
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
if (containers.Count == NumContainers)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
NodeId nodeId = expectedNMTokens[0].GetNodeId();
// NMToken is sent for the allocated container.
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId);
}
/// <exception cref="System.Exception"/>
public static MockAM LaunchAM(RMApp app, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
rm, MockNM nm)
{
rm.WaitForState(app.GetApplicationId(), RMAppState.Accepted);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
System.Console.Out.WriteLine("Launch AM " + attempt.GetAppAttemptId());
nm.NodeHeartbeat(true);
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
rm.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Launched);
return(am);
}
