public virtual void TestStateStoreAppLimitLargerThanMemoryAppLimit() { long now = Runtime.CurrentTimeMillis(); RMContext rmContext = MockRMContext(10, now - 20000); Configuration conf = new YarnConfiguration(); int maxAppsInMemory = 8; conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, maxAppsInMemory); // larger than maxCompletedAppsInMemory, reset to RM_MAX_COMPLETED_APPLICATIONS. conf.SetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications, 1000); TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager( this, rmContext, conf); AddToCompletedApps(appMonitor, rmContext); NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect", 10, appMonitor .GetCompletedAppsListSize()); appMonitor.CheckAppNumCompletedLimit(); int numRemoveApps = 10 - maxAppsInMemory; NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check" , maxAppsInMemory, rmContext.GetRMApps().Count); NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check", maxAppsInMemory, appMonitor.GetCompletedAppsListSize()); Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Times(numRemoveApps )).RemoveApplication(Matchers.IsA <RMApp>()); NUnit.Framework.Assert.AreEqual(maxAppsInMemory, appMonitor.GetCompletedAppsInStateStore ()); }
public virtual void TestRMAppRetireSomeDifferentStates() { long now = Runtime.CurrentTimeMillis(); // these parameters don't matter, override applications below RMContext rmContext = MockRMContext(10, now - 20000); Configuration conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications, 2); conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, 2); TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager( this, rmContext, conf); // clear out applications map rmContext.GetRMApps().Clear(); NUnit.Framework.Assert.AreEqual("map isn't empty", 0, rmContext.GetRMApps().Count ); // 6 applications are in final state, 4 are not in final state. // / set with various finished states RMApp app = new MockRMApp(0, now - 20000, RMAppState.Killed); rmContext.GetRMApps()[app.GetApplicationId()] = app; app = new MockRMApp(1, now - 200000, RMAppState.Failed); rmContext.GetRMApps()[app.GetApplicationId()] = app; app = new MockRMApp(2, now - 30000, RMAppState.Finished); rmContext.GetRMApps()[app.GetApplicationId()] = app; app = new MockRMApp(3, now - 20000, RMAppState.Running); rmContext.GetRMApps()[app.GetApplicationId()] = app; app = new MockRMApp(4, now - 20000, RMAppState.New); rmContext.GetRMApps()[app.GetApplicationId()] = app; // make sure it doesn't expire these since still running app = new MockRMApp(5, now - 10001, RMAppState.Killed); rmContext.GetRMApps()[app.GetApplicationId()] = app; app = new MockRMApp(6, now - 30000, RMAppState.Accepted); rmContext.GetRMApps()[app.GetApplicationId()] = app; app = new MockRMApp(7, now - 20000, RMAppState.Submitted); rmContext.GetRMApps()[app.GetApplicationId()] = app; app = new MockRMApp(8, now - 10001, RMAppState.Failed); rmContext.GetRMApps()[app.GetApplicationId()] = app; app = new MockRMApp(9, now - 20000, RMAppState.Failed); rmContext.GetRMApps()[app.GetApplicationId()] = app; NUnit.Framework.Assert.AreEqual("Number of apps incorrect before", 10, rmContext. GetRMApps().Count); // add them to completed apps list AddToCompletedApps(appMonitor, rmContext); // shouldn't have to many apps appMonitor.CheckAppNumCompletedLimit(); NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check" , 6, rmContext.GetRMApps().Count); NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check", 2, appMonitor.GetCompletedAppsListSize()); // 6 applications in final state, 4 of them are removed Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Times(4 )).RemoveApplication(Matchers.IsA <RMApp>()); }
protected internal virtual void AddToCompletedApps(TestAppManager.TestRMAppManager appMonitor, RMContext rmContext) { foreach (RMApp app in rmContext.GetRMApps().Values) { if (app.GetState() == RMAppState.Finished || app.GetState() == RMAppState.Killed || app.GetState() == RMAppState.Failed) { appMonitor.FinishApplication(app.GetApplicationId()); } } }
public virtual void TestRMAppRetireNullApp() { long now = Runtime.CurrentTimeMillis(); RMContext rmContext = MockRMContext(10, now - 20000); TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager( this, rmContext, new Configuration()); NUnit.Framework.Assert.AreEqual("Number of apps incorrect before", 10, rmContext. GetRMApps().Count); appMonitor.FinishApplication(null); NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check", 0, appMonitor.GetCompletedAppsListSize()); }
public virtual void SetUp() { long now = Runtime.CurrentTimeMillis(); rmContext = MockRMContext(1, now - 10); ResourceScheduler scheduler = MockResourceScheduler(); Configuration conf = new Configuration(); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); appMonitor = new TestAppManager.TestRMAppManager(this, rmContext, new ClientToAMTokenSecretManagerInRM (), scheduler, masterService, new ApplicationACLsManager(conf), conf); appId = MockApps.NewAppID(1); RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null); asContext = recordFactory.NewRecordInstance <ApplicationSubmissionContext>(); asContext.SetApplicationId(appId); asContext.SetAMContainerSpec(MockContainerLaunchContext(recordFactory)); asContext.SetResource(MockResource()); SetupDispatcher(rmContext, conf); }
/// <exception cref="System.Exception"/> public virtual void TestRMAppSubmitMaxAppAttempts() { int[] globalMaxAppAttempts = new int[] { 10, 1 }; int[][] individualMaxAppAttempts = new int[][] { new int[] { 9, 10, 11, 0 }, new int[] { 1, 10, 0, -1 } }; int[][] expectedNums = new int[][] { new int[] { 9, 10, 10, 10 }, new int[] { 1, 1, 1, 1 } }; for (int i = 0; i < globalMaxAppAttempts.Length; ++i) { for (int j = 0; j < individualMaxAppAttempts.Length; ++j) { ResourceScheduler scheduler = MockResourceScheduler(); Configuration conf = new Configuration(); conf.SetInt(YarnConfiguration.RmAmMaxAttempts, globalMaxAppAttempts[i]); ApplicationMasterService masterService = new ApplicationMasterService(rmContext, scheduler); TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager( this, rmContext, new ClientToAMTokenSecretManagerInRM(), scheduler, masterService , new ApplicationACLsManager(conf), conf); ApplicationId appID = MockApps.NewAppID(i * 4 + j + 1); asContext.SetApplicationId(appID); if (individualMaxAppAttempts[i][j] != 0) { asContext.SetMaxAppAttempts(individualMaxAppAttempts[i][j]); } appMonitor.SubmitApplication(asContext, "test"); RMApp app = rmContext.GetRMApps()[appID]; NUnit.Framework.Assert.AreEqual("max application attempts doesn't match", expectedNums [i][j], app.GetMaxAppAttempts()); // wait for event to be processed int timeoutSecs = 0; while ((GetAppEventType() == RMAppEventType.Kill) && timeoutSecs++ < 20) { Sharpen.Thread.Sleep(1000); } SetAppEventType(RMAppEventType.Kill); } } }
public virtual void TestRMAppRetireZeroSetting() { long now = Runtime.CurrentTimeMillis(); RMContext rmContext = MockRMContext(10, now - 20000); Configuration conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmStateStoreMaxCompletedApplications, 0); conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, 0); TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager( this, rmContext, conf); NUnit.Framework.Assert.AreEqual("Number of apps incorrect before", 10, rmContext. GetRMApps().Count); AddToCompletedApps(appMonitor, rmContext); NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect", 10, appMonitor .GetCompletedAppsListSize()); appMonitor.CheckAppNumCompletedLimit(); NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check" , 0, rmContext.GetRMApps().Count); NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check", 0, appMonitor.GetCompletedAppsListSize()); Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Times(10 )).RemoveApplication(Matchers.IsA <RMApp>()); }
public virtual void TestRMAppRetireNone() { long now = Runtime.CurrentTimeMillis(); // Create such that none of the applications will retire since // haven't hit max # RMContext rmContext = MockRMContext(10, now - 10); Configuration conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmMaxCompletedApplications, 10); TestAppManager.TestRMAppManager appMonitor = new TestAppManager.TestRMAppManager( this, rmContext, conf); NUnit.Framework.Assert.AreEqual("Number of apps incorrect before checkAppTimeLimit" , 10, rmContext.GetRMApps().Count); // add them to completed apps list AddToCompletedApps(appMonitor, rmContext); // shouldn't have to many apps appMonitor.CheckAppNumCompletedLimit(); NUnit.Framework.Assert.AreEqual("Number of apps incorrect after # completed check" , 10, rmContext.GetRMApps().Count); NUnit.Framework.Assert.AreEqual("Number of completed apps incorrect after check", 10, appMonitor.GetCompletedAppsListSize()); Org.Mockito.Mockito.Verify(rmContext.GetStateStore(), Org.Mockito.Mockito.Never() ).RemoveApplication(Matchers.IsA <RMApp>()); }