// There are two scenarios when RM failover happens
// during SubmitApplication Call:
// 1) RMStateStore already saved the ApplicationState when failover happens
// 2) RMStateStore did not save the ApplicationState when failover happens
/// <exception cref="System.Exception"/>
public virtual void TestHandleRMHADuringSubmitApplicationCallWithSavedApplicationState
()
{
// Test scenario 1 when RM failover happens
// druing SubmitApplication Call:
// RMStateStore already saved the ApplicationState when failover happens
StartRMs();
// Submit Application
// After submission, the applicationState will be saved in RMStateStore.
RMApp app0 = rm1.SubmitApp(200);
// Do the failover
ExplicitFailover();
// Since the applicationState has already been saved in RMStateStore
// before failover happens, the current active rm can load the previous
// applicationState.
// This RMApp should exist in the RMContext of current active RM
NUnit.Framework.Assert.IsTrue(rm2.GetRMContext().GetRMApps().Contains(app0.GetApplicationId
()));
// When we re-submit the application with same applicationId, it will
// check whether this application has been exist. If yes, just simply
// return submitApplicationResponse.
RMApp app1 = rm2.SubmitApp(200, string.Empty, UserGroupInformation.GetCurrentUser
().GetShortUserName(), null, false, null, configuration.GetInt(YarnConfiguration
.RmAmMaxAttempts, YarnConfiguration.DefaultRmAmMaxAttempts), null, null, false,
false, true, app0.GetApplicationId());
NUnit.Framework.Assert.AreEqual(app1.GetApplicationId(), app0.GetApplicationId());
}
/// <exception cref="System.Exception"/>
public virtual void TestAppCleanupWhenNMReconnects()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// start RM
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// create app and launch the AM
RMApp app0 = rm1.SubmitApp(200);
MockAM am0 = LaunchAM(app0, rm1, nm1);
nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Complete);
rm1.WaitForState(app0.GetApplicationId(), RMAppState.Failed);
// wait for application cleanup message received
WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
// reconnect NM with application still active
nm1.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
rm1.Stop();
}
// Test regular RM restart/failover, new RM should not count
// AM failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestRMRestartOrFailoverNotCountedForAMFailures()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
// AM should be restarted even though max-am-attempt is 1.
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt1).MayBeLastAttempt());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
// re-register the NM
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus
>();
status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager);
status.SetContainerId(attempt1.GetMasterContainer().GetId());
status.SetContainerState(ContainerState.Complete);
status.SetDiagnostics(string.Empty);
nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null);
rm2.WaitForState(attempt1.GetAppAttemptId(), RMAppAttemptState.Failed);
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledByResourcemanager, appState
.GetAttempt(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Will automatically start a new AppAttempt in rm2
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt3 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt3.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestKillAppWhenFailoverHappensAtNewState()
{
// create a customized RMAppManager
// During the process of Application submission,
// the RMAppState will always be NEW.
// The ApplicationState will not be saved in RMStateStore.
StartRMsWithCustomizedRMAppManager();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// Submit the application
RMApp app0 = rm1.SubmitApp(200, string.Empty, UserGroupInformation.GetCurrentUser
().GetShortUserName(), null, false, null, configuration.GetInt(YarnConfiguration
.RmAmMaxAttempts, YarnConfiguration.DefaultRmAmMaxAttempts), null, null, false,
false);
// failover and kill application
// When FailOver happens, the state of this application is NEW,
// and ApplicationState is not saved in RMStateStore. The active RM
// can not load the ApplicationState of this application.
// Expected to get ApplicationNotFoundException
// when receives the KillApplicationRequest
try
{
FailOverAndKillApp(app0.GetApplicationId(), RMAppState.New);
NUnit.Framework.Assert.Fail("Should get an exception here");
}
catch (ApplicationNotFoundException ex)
{
NUnit.Framework.Assert.IsTrue(ex.Message.Contains("Trying to kill an absent application "
+ app0.GetApplicationId()));
}
}
/// <exception cref="System.Exception"/>
private void VerifySubmitApp(MockRM rm, RMApp app, ApplicationId expectedAppId)
{
int maxWaittingTimes = 20;
int count = 0;
while (true)
{
YarnApplicationState state = rm.GetApplicationReport(app.GetApplicationId()).GetYarnApplicationState
();
if (!state.Equals(YarnApplicationState.New) && !state.Equals(YarnApplicationState
.NewSaving))
{
break;
}
if (count > maxWaittingTimes)
{
break;
}
Sharpen.Thread.Sleep(200);
count++;
}
// Verify submittion is successful
YarnApplicationState state_1 = rm.GetApplicationReport(app.GetApplicationId()).GetYarnApplicationState
();
NUnit.Framework.Assert.IsTrue(state_1 == YarnApplicationState.Accepted || state_1
== YarnApplicationState.Submitted);
NUnit.Framework.Assert.AreEqual(expectedAppId, app.GetApplicationId());
}
// Test RM restarts after AM container is preempted, new RM should not count
// AM preemption failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestPreemptedAMRestartOnRMRestart()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler();
ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 1);
// Forcibly preempt the am container;
scheduler.KillContainer(scheduler.GetRMContainer(amContainer));
am1.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry());
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// state store has 1 attempt stored.
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
NUnit.Framework.Assert.AreEqual(1, appState.GetAttemptCount());
// attempt stored has the preempted container exit status.
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Preempted, appState.GetAttempt
(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode();
rm2.Start();
// Restarted RM should re-launch the am.
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt2 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt2.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
public virtual void TestMoveAppToPlanQueue()
{
CapacityScheduler scheduler = (CapacityScheduler)rm.GetResourceScheduler();
// submit an app
RMApp app = rm.SubmitApp(Gb, "test-move-1", "user_0", null, "b1");
ApplicationAttemptId appAttemptId = rm.GetApplicationReport(app.GetApplicationId(
)).GetCurrentApplicationAttemptId();
// check preconditions
IList <ApplicationAttemptId> appsInB1 = scheduler.GetAppsInQueue("b1");
NUnit.Framework.Assert.AreEqual(1, appsInB1.Count);
IList <ApplicationAttemptId> appsInB = scheduler.GetAppsInQueue("b");
NUnit.Framework.Assert.AreEqual(1, appsInB.Count);
NUnit.Framework.Assert.IsTrue(appsInB.Contains(appAttemptId));
IList <ApplicationAttemptId> appsInA = scheduler.GetAppsInQueue("a");
NUnit.Framework.Assert.IsTrue(appsInA.IsEmpty());
string queue = scheduler.GetApplicationAttempt(appsInB1[0]).GetQueue().GetQueueName
();
NUnit.Framework.Assert.IsTrue(queue.Equals("b1"));
IList <ApplicationAttemptId> appsInRoot = scheduler.GetAppsInQueue("root");
NUnit.Framework.Assert.IsTrue(appsInRoot.Contains(appAttemptId));
NUnit.Framework.Assert.AreEqual(1, appsInRoot.Count);
// create the default reservation queue
string defQName = "a" + ReservationConstants.DefaultQueueSuffix;
ReservationQueue defQ = new ReservationQueue(scheduler, defQName, (PlanQueue)scheduler
.GetQueue("a"));
scheduler.AddQueue(defQ);
defQ.SetEntitlement(new QueueEntitlement(1f, 1f));
IList <ApplicationAttemptId> appsInDefQ = scheduler.GetAppsInQueue(defQName);
NUnit.Framework.Assert.IsTrue(appsInDefQ.IsEmpty());
// now move the app to plan queue
scheduler.MoveApplication(app.GetApplicationId(), "a");
// check postconditions
appsInDefQ = scheduler.GetAppsInQueue(defQName);
NUnit.Framework.Assert.AreEqual(1, appsInDefQ.Count);
queue = scheduler.GetApplicationAttempt(appsInDefQ[0]).GetQueue().GetQueueName();
NUnit.Framework.Assert.IsTrue(queue.Equals(defQName));
appsInA = scheduler.GetAppsInQueue("a");
NUnit.Framework.Assert.IsTrue(appsInA.Contains(appAttemptId));
NUnit.Framework.Assert.AreEqual(1, appsInA.Count);
appsInRoot = scheduler.GetAppsInQueue("root");
NUnit.Framework.Assert.IsTrue(appsInRoot.Contains(appAttemptId));
NUnit.Framework.Assert.AreEqual(1, appsInRoot.Count);
appsInB1 = scheduler.GetAppsInQueue("b1");
NUnit.Framework.Assert.IsTrue(appsInB1.IsEmpty());
appsInB = scheduler.GetAppsInQueue("b");
NUnit.Framework.Assert.IsTrue(appsInB.IsEmpty());
rm.Stop();
}
// This is to test AM Host and rpc port are invalidated after the am attempt
// is killed or failed, so that client doesn't get the wrong information.
/// <exception cref="System.Exception"/>
public virtual void TestInvalidateAMHostPortWhenAMFailedOrKilled()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MockRM rm1 = new MockRM(conf);
rm1.Start();
// a succeeded app
RMApp app1 = rm1.SubmitApp(200);
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm1, nm1, am1);
// a failed app
RMApp app2 = rm1.SubmitApp(200);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app2.GetApplicationId(), RMAppState.Failed);
// a killed app
RMApp app3 = rm1.SubmitApp(200);
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm1);
rm1.KillApp(app3.GetApplicationId());
rm1.WaitForState(app3.GetApplicationId(), RMAppState.Killed);
rm1.WaitForState(am3.GetApplicationAttemptId(), RMAppAttemptState.Killed);
GetApplicationsRequest request1 = GetApplicationsRequest.NewInstance(EnumSet.Of(YarnApplicationState
.Finished, YarnApplicationState.Killed, YarnApplicationState.Failed));
GetApplicationsResponse response1 = rm1.GetClientRMService().GetApplications(request1
);
IList <ApplicationReport> appList1 = response1.GetApplicationList();
NUnit.Framework.Assert.AreEqual(3, appList1.Count);
foreach (ApplicationReport report in appList1)
{
// killed/failed apps host and rpc port are invalidated.
if (report.GetApplicationId().Equals(app2.GetApplicationId()) || report.GetApplicationId
().Equals(app3.GetApplicationId()))
{
NUnit.Framework.Assert.AreEqual("N/A", report.GetHost());
NUnit.Framework.Assert.AreEqual(-1, report.GetRpcPort());
}
// succeeded app's host and rpc port is not invalidated
if (report.GetApplicationId().Equals(app1.GetApplicationId()))
{
NUnit.Framework.Assert.IsFalse(report.GetHost().Equals("N/A"));
NUnit.Framework.Assert.IsTrue(report.GetRpcPort() != -1);
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAppCleanupWhenRMRestartedBeforeAppFinished()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// start RM
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 1024, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockNM nm2 = new MockNM("127.0.0.1:5678", 1024, rm1.GetResourceTrackerService());
nm2.RegisterNode();
// create app and launch the AM
RMApp app0 = rm1.SubmitApp(200);
MockAM am0 = LaunchAM(app0, rm1, nm1);
// alloc another container on nm2
AllocateResponse allocResponse = am0.Allocate(Arrays.AsList(ResourceRequest.NewInstance
(Priority.NewInstance(1), "*", Resource.NewInstance(1024, 0), 1)), null);
while (null == allocResponse.GetAllocatedContainers() || allocResponse.GetAllocatedContainers
().IsEmpty())
{
nm2.NodeHeartbeat(true);
allocResponse = am0.Allocate(null, null);
Sharpen.Thread.Sleep(1000);
}
// start new RM
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
// nm1/nm2 register to rm2, and do a heartbeat
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode(Arrays.AsList(NMContainerStatus.NewInstance(ContainerId.NewContainerId
(am0.GetApplicationAttemptId(), 1), ContainerState.Complete, Resource.NewInstance
(1024, 1), string.Empty, 0, Priority.NewInstance(0), 1234)), Arrays.AsList(app0.
GetApplicationId()));
nm2.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm2.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
// assert app state has been saved.
rm2.WaitForState(app0.GetApplicationId(), RMAppState.Failed);
// wait for application cleanup message received on NM1
WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
// wait for application cleanup message received on NM2
WaitForAppCleanupMessageRecved(nm2, app0.GetApplicationId());
rm1.Stop();
rm2.Stop();
}
/*
* check to see if hit the limit for max # completed apps kept
*/
protected internal virtual void CheckAppNumCompletedLimit()
{
lock (this)
{
// check apps kept in state store.
while (completedAppsInStateStore > this.maxCompletedAppsInStateStore)
{
ApplicationId removeId = completedApps[completedApps.Count - completedAppsInStateStore
];
RMApp removeApp = rmContext.GetRMApps()[removeId];
Log.Info("Max number of completed apps kept in state store met:" + " maxCompletedAppsInStateStore = "
+ maxCompletedAppsInStateStore + ", removing app " + removeApp.GetApplicationId
() + " from state store.");
rmContext.GetStateStore().RemoveApplication(removeApp);
completedAppsInStateStore--;
}
// check apps kept in memorty.
while (completedApps.Count > this.maxCompletedAppsInMemory)
{
ApplicationId removeId = completedApps.Remove();
Log.Info("Application should be expired, max number of completed apps" + " kept in memory met: maxCompletedAppsInMemory = "
+ this.maxCompletedAppsInMemory + ", removing app " + removeId + " from memory: "
);
Sharpen.Collections.Remove(rmContext.GetRMApps(), removeId);
this.applicationACLsManager.RemoveApplication(removeId);
}
}
}
private static RMApp CreateRMApp(ApplicationId appId)
{
RMApp app = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(appId);
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("test app");
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("test app type");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("test user");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("test queue");
Org.Mockito.Mockito.When(app.GetSubmitTime()).ThenReturn(int.MaxValue + 1L);
Org.Mockito.Mockito.When(app.GetStartTime()).ThenReturn(int.MaxValue + 2L);
Org.Mockito.Mockito.When(app.GetFinishTime()).ThenReturn(int.MaxValue + 3L);
Org.Mockito.Mockito.When(app.GetDiagnostics()).ThenReturn(new StringBuilder("test diagnostics info"
));
RMAppAttempt appAttempt = Org.Mockito.Mockito.Mock <RMAppAttempt>();
Org.Mockito.Mockito.When(appAttempt.GetAppAttemptId()).ThenReturn(ApplicationAttemptId
.NewInstance(appId, 1));
Org.Mockito.Mockito.When(app.GetCurrentAppAttempt()).ThenReturn(appAttempt);
Org.Mockito.Mockito.When(app.GetFinalApplicationStatus()).ThenReturn(FinalApplicationStatus
.Undefined);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(new RMAppMetrics(null,
0, 0, int.MaxValue, long.MaxValue));
return(app);
}
/// <exception cref="System.IO.IOException"/>
public virtual bool AppExists(RMApp app)
{
FileSystem fs = this.cluster.GetFileSystem();
Path nodePath = this.store.GetAppDir(app.GetApplicationId().ToString());
return(fs.Exists(nodePath));
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidatedAMHostPortOnAMRestart()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// a failed app
RMApp app2 = rm1.SubmitApp(200);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app2.GetApplicationId(), RMAppState.Accepted);
// before new attempt is launched, the app report returns the invalid AM
// host and port.
GetApplicationReportRequest request1 = GetApplicationReportRequest.NewInstance(app2
.GetApplicationId());
ApplicationReport report1 = rm1.GetClientRMService().GetApplicationReport(request1
).GetApplicationReport();
NUnit.Framework.Assert.AreEqual("N/A", report1.GetHost());
NUnit.Framework.Assert.AreEqual(-1, report1.GetRpcPort());
}
/// <exception cref="System.Exception"/>
public virtual bool AppExists(RMApp app)
{
Stat node = this.client.Exists(this.store.GetAppNode(app.GetApplicationId().ToString
()), false);
return(node != null);
}
/// <exception cref="System.Exception"/>
public virtual void TestRMAppSubmitDuplicateApplicationId()
{
ApplicationId appId = MockApps.NewAppID(0);
asContext.SetApplicationId(appId);
RMApp appOrig = rmContext.GetRMApps()[appId];
NUnit.Framework.Assert.IsTrue("app name matches but shouldn't", "testApp1" != appOrig
.GetName());
// our testApp1 should be rejected and original app with same id should be left in place
try
{
appMonitor.SubmitApplication(asContext, "test");
NUnit.Framework.Assert.Fail("Exception is expected when applicationId is duplicate."
);
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsTrue("The thrown exception is not the expectd one.", e.Message
.Contains("Cannot add a duplicate!"));
}
// make sure original app didn't get removed
RMApp app = rmContext.GetRMApps()[appId];
NUnit.Framework.Assert.IsNotNull("app is null", app);
NUnit.Framework.Assert.AreEqual("app id doesn't match", appId, app.GetApplicationId
());
NUnit.Framework.Assert.AreEqual("app state doesn't match", RMAppState.Finished, app
.GetState());
}
/// <exception cref="System.Exception"/>
public virtual void TestMoveSuccessful()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
RMApp app = rm1.SubmitApp(1024);
ClientRMService clientRMService = rm1.GetClientRMService();
// FIFO scheduler does not support moves
clientRMService.MoveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest.NewInstance
(app.GetApplicationId(), "newqueue"));
RMApp rmApp = rm1.GetRMContext().GetRMApps()[app.GetApplicationId()];
NUnit.Framework.Assert.AreEqual("newqueue", rmApp.GetQueue());
rm1.Stop();
}
public virtual void TestEscapeApplicationSummary()
{
RMApp app = Org.Mockito.Mockito.Mock <RMAppImpl>();
Org.Mockito.Mockito.When(app.GetApplicationId()).ThenReturn(ApplicationId.NewInstance
(100L, 1));
Org.Mockito.Mockito.When(app.GetName()).ThenReturn("Multiline\n\n\r\rAppName");
Org.Mockito.Mockito.When(app.GetUser()).ThenReturn("Multiline\n\n\r\rUserName");
Org.Mockito.Mockito.When(app.GetQueue()).ThenReturn("Multiline\n\n\r\rQueueName");
Org.Mockito.Mockito.When(app.GetState()).ThenReturn(RMAppState.Running);
Org.Mockito.Mockito.When(app.GetApplicationType()).ThenReturn("MAPREDUCE");
RMAppMetrics metrics = new RMAppMetrics(Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1234, 56), 10, 1, 16384, 64);
Org.Mockito.Mockito.When(app.GetRMAppMetrics()).ThenReturn(metrics);
RMAppManager.ApplicationSummary.SummaryBuilder summary = RMAppManager.ApplicationSummary
.CreateAppSummary(app);
string msg = summary.ToString();
Log.Info("summary: " + msg);
NUnit.Framework.Assert.IsFalse(msg.Contains("\n"));
NUnit.Framework.Assert.IsFalse(msg.Contains("\r"));
string escaped = "\\n\\n\\r\\r";
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "AppName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "UserName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("Multiline" + escaped + "QueueName"));
NUnit.Framework.Assert.IsTrue(msg.Contains("memorySeconds=16384"));
NUnit.Framework.Assert.IsTrue(msg.Contains("vcoreSeconds=64"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedAMContainers=1"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedNonAMContainers=10"));
NUnit.Framework.Assert.IsTrue(msg.Contains("preemptedResources=<memory:1234\\, vCores:56>"
));
NUnit.Framework.Assert.IsTrue(msg.Contains("applicationType=MAPREDUCE"));
}
/// <exception cref="System.Exception"/>
public virtual bool AppExists(RMApp app)
{
if (this._enclosing.stateStore.IsClosed())
{
this.GetRMStateStore();
}
return(this._enclosing.stateStore.LoadRMAppState(app.GetApplicationId()) != null);
}
public virtual void AppACLsUpdated(RMApp app, string appViewACLs, long updatedTime
)
{
if (publishSystemMetrics)
{
dispatcher.GetEventHandler().Handle(new ApplicationACLsUpdatedEvent(app.GetApplicationId
(), appViewACLs == null ? string.Empty : appViewACLs, updatedTime));
}
}
public virtual void AppCreated(RMApp app, long createdTime)
{
if (publishSystemMetrics)
{
dispatcher.GetEventHandler().Handle(new ApplicationCreatedEvent(app.GetApplicationId
(), app.GetName(), app.GetApplicationType(), app.GetUser(), app.GetQueue(), app.
GetSubmitTime(), createdTime));
}
}
/// <exception cref="System.Exception"/>
public static MockAM LaunchAndRegisterAM(RMApp app, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
rm, MockNM nm)
{
MockAM am = LaunchAM(app, rm, nm);
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
return(am);
}
/// <summary>Validate killing an application when it is at accepted state.</summary>
/// <exception cref="System.Exception">exception</exception>
public virtual void TestApplicationKillAtAcceptedState()
{
Dispatcher dispatcher = new _AsyncDispatcher_573();
MockRM rm = new _MockRM_596(dispatcher, conf);
// test metrics
QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
int appsKilled = metrics.GetAppsKilled();
int appsSubmitted = metrics.GetAppsSubmitted();
rm.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());
nm1.RegisterNode();
// a failed app
RMApp application = rm.SubmitApp(200);
MockAM am = MockRM.LaunchAM(application, rm, nm1);
am.WaitForState(RMAppAttemptState.Launched);
nm1.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Running);
rm.WaitForState(application.GetApplicationId(), RMAppState.Accepted);
// Now kill the application before new attempt is launched, the app report
// returns the invalid AM host and port.
KillApplicationRequest request = KillApplicationRequest.NewInstance(application.GetApplicationId
());
rm.GetClientRMService().ForceKillApplication(request);
// Specific test for YARN-1689 follows
// Now let's say a race causes AM to register now. This should not crash RM.
am.RegisterAppAttempt(false);
// We explicitly intercepted the kill-event to RMAppAttempt, so app should
// still be in KILLING state.
rm.WaitForState(application.GetApplicationId(), RMAppState.Killing);
// AM should now be in running
rm.WaitForState(am.GetApplicationAttemptId(), RMAppAttemptState.Running);
// Simulate that appAttempt is killed.
rm.GetRMContext().GetDispatcher().GetEventHandler().Handle(new RMAppEvent(application
.GetApplicationId(), RMAppEventType.AttemptKilled));
rm.WaitForState(application.GetApplicationId(), RMAppState.Killed);
// test metrics
metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
NUnit.Framework.Assert.AreEqual(appsKilled + 1, metrics.GetAppsKilled());
NUnit.Framework.Assert.AreEqual(appsSubmitted + 1, metrics.GetAppsSubmitted());
}
public virtual void AppFinished(RMApp app, RMAppState state, long finishedTime)
{
if (publishSystemMetrics)
{
dispatcher.GetEventHandler().Handle(new ApplicationFinishedEvent(app.GetApplicationId
(), app.GetDiagnostics().ToString(), app.GetFinalApplicationStatus(), RMServerUtils
.CreateApplicationState(state), app.GetCurrentAppAttempt() == null ? null : app.
GetCurrentAppAttempt().GetAppAttemptId(), finishedTime, app.GetRMAppMetrics()));
}
}
public virtual void ApplicationStarted(RMApp app)
{
if (historyServiceEnabled)
{
dispatcher.GetEventHandler().Handle(new WritingApplicationStartEvent(app.GetApplicationId
(), ApplicationStartData.NewInstance(app.GetApplicationId(), app.GetName(), app.
GetApplicationType(), app.GetQueue(), app.GetUser(), app.GetSubmitTime(), app.GetStartTime
())));
}
}
public virtual void TestHandleRMHAafterSubmitApplicationCallWithoutSavedApplicationState
()
{
// Test scenario 2 when RM failover happens
// after SubmitApplication Call:
// RMStateStore did not save the ApplicationState when failover happens.
// Using customized RMAppManager.
StartRMsWithCustomizedRMAppManager();
// Submit Application
// After submission, the applicationState will
// not be saved in RMStateStore
RMApp app0 = rm1.SubmitApp(200, string.Empty, UserGroupInformation.GetCurrentUser
().GetShortUserName(), null, false, null, configuration.GetInt(YarnConfiguration
.RmAmMaxAttempts, YarnConfiguration.DefaultRmAmMaxAttempts), null, null, false,
false);
// Do the failover
ExplicitFailover();
// Since the applicationState is not saved in RMStateStore
// when failover happens. The current active RM can not load
// previous applicationState.
// Expect ApplicationNotFoundException by calling getApplicationReport().
try
{
rm2.GetApplicationReport(app0.GetApplicationId());
NUnit.Framework.Assert.Fail("Should get ApplicationNotFoundException here");
}
catch (ApplicationNotFoundException)
{
}
// expected ApplicationNotFoundException
// Submit the application with previous ApplicationId to current active RM
// This will mimic the similar behavior of YarnClient which will re-submit
// Application with previous applicationId
// when catches the ApplicationNotFoundException
RMApp app1 = rm2.SubmitApp(200, string.Empty, UserGroupInformation.GetCurrentUser
().GetShortUserName(), null, false, null, configuration.GetInt(YarnConfiguration
.RmAmMaxAttempts, YarnConfiguration.DefaultRmAmMaxAttempts), null, null, false,
false, true, app0.GetApplicationId());
VerifySubmitApp(rm2, app1, app0.GetApplicationId());
}
// Test Kill an app while the app is failing
/// <exception cref="System.Exception"/>
public virtual void TestKillFailingApp()
{
// this dispatcher ignores RMAppAttemptEventType.KILL event
Dispatcher dispatcher = new _AsyncDispatcher_708();
MockRM rm1 = new _MockRM_731(dispatcher, conf);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
rm1.KillApp(app1.GetApplicationId());
// fail the app by sending container_finished event.
nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete);
rm1.WaitForState(am1.GetApplicationAttemptId(), RMAppAttemptState.Failed);
// app is killed, not launching a new attempt
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Killed);
}
/// <exception cref="System.Exception"/>
private MockAM LaunchAM(RMApp app, MockRM rm, MockNM nm)
{
RMAppAttempt attempt = app.GetCurrentAppAttempt();
nm.NodeHeartbeat(true);
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
return(am);
}
// Disable webapp
/// <exception cref="System.Exception"/>
public static void FinishAMAndVerifyAppState(RMApp rmApp, Org.Apache.Hadoop.Yarn.Server.Resourcemanager.MockRM
rm, MockNM nm, MockAM am)
{
FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
.Succeeded, string.Empty, string.Empty);
am.UnregisterAppAttempt(req, true);
am.WaitForState(RMAppAttemptState.Finishing);
nm.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.WaitForState(rmApp.GetApplicationId(), RMAppState.Finished);
}
// Test even if AM container is allocated with containerId not equal to 1, the
// following allocate requests from AM should be able to retrieve the
// corresponding NM Token.
/// <exception cref="System.Exception"/>
public virtual void TestNMTokenSentForNormalContainer()
{
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName
());
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
// Call getNewContainerId to increase container Id so that the AM container
// Id doesn't equal to one.
CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId();
// kick the scheduling
nm1.NodeHeartbeat(true);
MockAM am = MockRM.LaunchAM(app, rm, nm1);
// am container Id not equal to 1.
NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId
() != 1);
// NMSecretManager doesn't record the node on which the am is allocated.
NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
int NumContainers = 1;
IList <Container> containers = new AList <Container>();
// nmTokens keeps track of all the nmTokens issued in the allocate call.
IList <NMToken> expectedNMTokens = new AList <NMToken>();
// am1 allocate 1 container on nm1.
while (true)
{
AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList
<ContainerId>());
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
if (containers.Count == NumContainers)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
NodeId nodeId = expectedNMTokens[0].GetNodeId();
// NMToken is sent for the allocated container.
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId);
}
/// <exception cref="System.Exception"/>
private void CheckAppQueue(MockRM resourceManager, string user, string submissionQueue
, string expected)
{
RMApp app = resourceManager.SubmitApp(200, "name", user, new Dictionary <ApplicationAccessType
, string>(), false, submissionQueue, -1, null, "MAPREDUCE", false);
RMAppState expectedState = expected.IsEmpty() ? RMAppState.Failed : RMAppState.Accepted;
resourceManager.WaitForState(app.GetApplicationId(), expectedState);
// get scheduler app
CapacityScheduler cs = (CapacityScheduler)resourceManager.GetResourceScheduler();
SchedulerApplication schedulerApp = cs.GetSchedulerApplications()[app.GetApplicationId
()];
string queue = string.Empty;
if (schedulerApp != null)
{
queue = schedulerApp.GetQueue().GetQueueName();
}
NUnit.Framework.Assert.IsTrue("expected " + expected + " actual " + queue, expected
.Equals(queue));
NUnit.Framework.Assert.AreEqual(expected, app.GetQueue());
}
