public virtual void TestStateStoreAppLimitLargerThanMemoryAppLimit()
{
long now = Runtime.CurrentTimeMillis();
RMContext rmContext = MockRMContext(10, now - 20000);
Configuration conf = new YarnConfiguration();
int maxAppsInMemory = 8;
conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, maxAppsInMemory);
// larger than maxCompletedAppsInMemory, reset to RM_MAX_COMPLETED_APPLICATIONS.
conf.SetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications, 1000);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, conf);
AddToCompletedApps(appMonitor, rmContext);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect", 10, appMonitor
.GetCompletedAppsListSize());
appMonitor.CheckAppNumCompletedLimit();
int numRemoveApps = 10 - maxAppsInMemory;
NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check"
, maxAppsInMemory, rmContext.GetRMApps().Count);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check",
maxAppsInMemory, appMonitor.GetCompletedAppsListSize());
Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Times(numRemoveApps
)).RemoveApplication(Matchers.IsA <RMApp>());
NUnit.Framework.Assert.AreEqual(maxAppsInMemory, appMonitor.GetCompletedAppsInStateStore
());
}
public virtual void TestRMAppRetireSomeDifferentStates()
{
long now = Runtime.CurrentTimeMillis();
// these parameters don't matter, override applications below
RMContext rmContext = MockRMContext(10, now - 20000);
Configuration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications, 2);
conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, 2);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, conf);
// clear out applications map
rmContext.GetRMApps().Clear();
NUnit.Framework.Assert.AreEqual("map isn't empty", 0, rmContext.GetRMApps().Count
);
// 6 applications are in final state, 4 are not in final state.
// / set with various finished states
RMApp app = new MockRMApp(0, now - 20000, RMAppState.Killed);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(1, now - 200000, RMAppState.Failed);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(2, now - 30000, RMAppState.Finished);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(3, now - 20000, RMAppState.Running);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(4, now - 20000, RMAppState.New);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
// make sure it doesn't expire these since still running
app = new MockRMApp(5, now - 10001, RMAppState.Killed);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(6, now - 30000, RMAppState.Accepted);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(7, now - 20000, RMAppState.Submitted);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(8, now - 10001, RMAppState.Failed);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
app = new MockRMApp(9, now - 20000, RMAppState.Failed);
rmContext.GetRMApps()[app.GetApplicationId()] = app;
NUnit.Framework.Assert.AreEqual("Number of apps incorrect before", 10, rmContext.
GetRMApps().Count);
// add them to completed apps list
AddToCompletedApps(appMonitor, rmContext);
// shouldn't have to many apps
appMonitor.CheckAppNumCompletedLimit();
NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check"
, 6, rmContext.GetRMApps().Count);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check",
2, appMonitor.GetCompletedAppsListSize());
// 6 applications in final state, 4 of them are removed
Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Times(4
)).RemoveApplication(Matchers.IsA <RMApp>());
}
protected internal virtual void AddToCompletedApps(TestAppManager.TestRMAppManager
appMonitor, RMContext rmContext)
{
foreach (RMApp app in rmContext.GetRMApps().Values)
{
if (app.GetState() == RMAppState.Finished || app.GetState() == RMAppState.Killed ||
app.GetState() == RMAppState.Failed)
{
appMonitor.FinishApplication(app.GetApplicationId());
}
}
}
public virtual void TestRMAppRetireNullApp()
{
long now = Runtime.CurrentTimeMillis();
RMContext rmContext = MockRMContext(10, now - 20000);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, new Configuration());
NUnit.Framework.Assert.AreEqual("Number of apps incorrect before", 10, rmContext.
GetRMApps().Count);
appMonitor.FinishApplication(null);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check",
0, appMonitor.GetCompletedAppsListSize());
}
public virtual void SetUp()
{
long now = Runtime.CurrentTimeMillis();
rmContext = MockRMContext(1, now - 10);
ResourceScheduler scheduler = MockResourceScheduler();
Configuration conf = new Configuration();
ApplicationMasterService masterService = new ApplicationMasterService(rmContext,
scheduler);
appMonitor = new TestAppManager.TestRMAppManager(this, rmContext, new ClientToAMTokenSecretManagerInRM
(), scheduler, masterService, new ApplicationACLsManager(conf), conf);
appId = MockApps.NewAppID(1);
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
asContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext>();
asContext.SetApplicationId(appId);
asContext.SetAMContainerSpec(MockContainerLaunchContext(recordFactory));
asContext.SetResource(MockResource());
SetupDispatcher(rmContext, conf);
}
/// <exception cref="System.Exception"/>
public virtual void TestRMAppSubmitMaxAppAttempts()
{
int[] globalMaxAppAttempts = new int[] { 10, 1 };
int[][] individualMaxAppAttempts = new int[][] { new int[] { 9, 10, 11, 0 }, new
int[] { 1, 10, 0, -1 } };
int[][] expectedNums = new int[][] { new int[] { 9, 10, 10, 10 }, new int[] { 1,
1, 1, 1 } };
for (int i = 0; i < globalMaxAppAttempts.Length; ++i)
{
for (int j = 0; j < individualMaxAppAttempts.Length; ++j)
{
ResourceScheduler scheduler = MockResourceScheduler();
Configuration conf = new Configuration();
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, globalMaxAppAttempts[i]);
ApplicationMasterService masterService = new ApplicationMasterService(rmContext,
scheduler);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, new ClientToAMTokenSecretManagerInRM(), scheduler, masterService
, new ApplicationACLsManager(conf), conf);
ApplicationId appID = MockApps.NewAppID(i * 4 + j + 1);
asContext.SetApplicationId(appID);
if (individualMaxAppAttempts[i][j] != 0)
{
asContext.SetMaxAppAttempts(individualMaxAppAttempts[i][j]);
}
appMonitor.SubmitApplication(asContext, "test");
RMApp app = rmContext.GetRMApps()[appID];
NUnit.Framework.Assert.AreEqual("max application attempts doesn't match", expectedNums
[i][j], app.GetMaxAppAttempts());
// wait for event to be processed
int timeoutSecs = 0;
while ((GetAppEventType() == RMAppEventType.Kill) && timeoutSecs++ < 20)
{
Sharpen.Thread.Sleep(1000);
}
SetAppEventType(RMAppEventType.Kill);
}
}
}
public virtual void TestRMAppRetireZeroSetting()
{
long now = Runtime.CurrentTimeMillis();
RMContext rmContext = MockRMContext(10, now - 20000);
Configuration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications, 0);
conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, 0);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, conf);
NUnit.Framework.Assert.AreEqual("Number of apps incorrect before", 10, rmContext.
GetRMApps().Count);
AddToCompletedApps(appMonitor, rmContext);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect", 10, appMonitor
.GetCompletedAppsListSize());
appMonitor.CheckAppNumCompletedLimit();
NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check"
, 0, rmContext.GetRMApps().Count);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check",
0, appMonitor.GetCompletedAppsListSize());
Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Times(10
)).RemoveApplication(Matchers.IsA <RMApp>());
}
public virtual void TestRMAppRetireNone()
{
long now = Runtime.CurrentTimeMillis();
// Create such that none of the applications will retire since
// haven't hit max #
RMContext rmContext = MockRMContext(10, now - 10);
Configuration conf = new YarnConfiguration();
conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, 10);
TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager(
this, rmContext, conf);
NUnit.Framework.Assert.AreEqual("Number of apps incorrect before checkAppTimeLimit"
, 10, rmContext.GetRMApps().Count);
// add them to completed apps list
AddToCompletedApps(appMonitor, rmContext);
// shouldn't have to many apps
appMonitor.CheckAppNumCompletedLimit();
NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check"
, 10, rmContext.GetRMApps().Count);
NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check",
10, appMonitor.GetCompletedAppsListSize());
Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Never()
).RemoveApplication(Matchers.IsA <RMApp>());
}