public virtual void TestHAIDLookup() { //test implicitly lookup HA-ID Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.AreEqual(conf.Get(YarnConfiguration.RmHaId), Rm2NodeId); //test explicitly lookup HA-ID configuration.Set(YarnConfiguration.RmHaId, Rm1NodeId); conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.AreEqual(conf.Get(YarnConfiguration.RmHaId), Rm1NodeId); //test if RM_HA_ID can not be found configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm3NodeId); configuration.Unset(YarnConfiguration.RmHaId); conf = new YarnConfiguration(configuration); try { rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.Fail("Should get an exception here."); } catch (Exception ex) { NUnit.Framework.Assert.IsTrue(ex.Message.Contains("Invalid configuration! Can not find valid RM_HA_ID." )); } }
/// <exception cref="System.Exception"/> public virtual void TestTransitionedToStandbyShouldNotHang() { configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); Configuration conf = new YarnConfiguration(configuration); MemoryRMStateStore memStore = new _MemoryRMStateStore_464(); memStore.Init(conf); rm = new _MockRM_472(conf, memStore); rm.Init(conf); HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing , rm.adminService.GetServiceStatus().GetState()); NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started" , rm.adminService.GetServiceStatus().IsReadyToBecomeActive()); CheckMonitorHealth(); rm.Start(); CheckMonitorHealth(); CheckStandbyRMFunctionality(); // 2. Transition to Active. rm.adminService.TransitionToActive(requestInfo); // 3. Try Transition to standby Sharpen.Thread t = new Sharpen.Thread(new _Runnable_498(this)); // TODO Auto-generated catch block t.Start(); rm.GetRMContext().GetStateStore().UpdateApplicationState(null); t.Join(); // wait for thread to finish rm.adminService.TransitionToStandby(requestInfo); CheckStandbyRMFunctionality(); rm.Stop(); }
public virtual void TestAdminRefreshQueuesWithFileSystemBasedConfigurationProvider () { configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); //upload default configurations UploadDefaultConfiguration(); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } CapacityScheduler cs = (CapacityScheduler)rm.GetRMContext().GetScheduler(); int maxAppsBefore = cs.GetConfiguration().GetMaximumSystemApplications(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.Set("yarn.scheduler.capacity.maximum-applications", "5000"); UploadConfiguration(csConf, "capacity-scheduler.xml"); rm.adminService.RefreshQueues(RefreshQueuesRequest.NewInstance()); int maxAppsAfter = cs.GetConfiguration().GetMaximumSystemApplications(); NUnit.Framework.Assert.AreEqual(maxAppsAfter, 5000); NUnit.Framework.Assert.IsTrue(maxAppsAfter != maxAppsBefore); }
public virtual void TestAdminAclsWithFileSystemBasedConfigurationProvider() { configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); //upload default configurations UploadDefaultConfiguration(); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } string aclStringBefore = rm.adminService.GetAccessControlList().GetAclString().Trim (); YarnConfiguration yarnConf = new YarnConfiguration(); yarnConf.Set(YarnConfiguration.YarnAdminAcl, "world:anyone:rwcda"); UploadConfiguration(yarnConf, "yarn-site.xml"); rm.adminService.RefreshAdminAcls(RefreshAdminAclsRequest.NewInstance()); string aclStringAfter = rm.adminService.GetAccessControlList().GetAclString().Trim (); NUnit.Framework.Assert.IsTrue(!aclStringAfter.Equals(aclStringBefore)); NUnit.Framework.Assert.AreEqual(aclStringAfter, "world:anyone:rwcda," + UserGroupInformation .GetCurrentUser().GetShortUserName()); }
// Other stuff is verified in the regular web-services related tests /// <summary>Test to verify the following RM HA transitions to the following states.</summary> /// <remarks> /// Test to verify the following RM HA transitions to the following states. /// 1. Standby: Should be a no-op /// 2. Active: Active services should start /// 3. Active: Should be a no-op. /// While active, submit a couple of jobs /// 4. Standby: Active services should stop /// 5. Active: Active services should start /// 6. Stop the RM: All services should stop and RM should not be ready to /// become Active /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestFailoverAndTransitions() { configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing , rm.adminService.GetServiceStatus().GetState()); NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started" , rm.adminService.GetServiceStatus().IsReadyToBecomeActive()); CheckMonitorHealth(); rm.Start(); CheckMonitorHealth(); CheckStandbyRMFunctionality(); VerifyClusterMetrics(0, 0, 0, 0, 0, 0); // 1. Transition to Standby - must be a no-op rm.adminService.TransitionToStandby(requestInfo); CheckMonitorHealth(); CheckStandbyRMFunctionality(); VerifyClusterMetrics(0, 0, 0, 0, 0, 0); // 2. Transition to active rm.adminService.TransitionToActive(requestInfo); CheckMonitorHealth(); CheckActiveRMFunctionality(); VerifyClusterMetrics(1, 1, 1, 1, 2048, 1); // 3. Transition to active - no-op rm.adminService.TransitionToActive(requestInfo); CheckMonitorHealth(); CheckActiveRMFunctionality(); VerifyClusterMetrics(1, 2, 2, 2, 2048, 2); // 4. Transition to standby rm.adminService.TransitionToStandby(requestInfo); CheckMonitorHealth(); CheckStandbyRMFunctionality(); VerifyClusterMetrics(0, 0, 0, 0, 0, 0); // 5. Transition to active to check Active->Standby->Active works rm.adminService.TransitionToActive(requestInfo); CheckMonitorHealth(); CheckActiveRMFunctionality(); VerifyClusterMetrics(1, 1, 1, 1, 2048, 1); // 6. Stop the RM. All services should stop and RM should not be ready to // become active rm.Stop(); NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Stopping , rm.adminService.GetServiceStatus().GetState()); NUnit.Framework.Assert.IsFalse("RM is ready to become active even after it is stopped" , rm.adminService.GetServiceStatus().IsReadyToBecomeActive()); NUnit.Framework.Assert.IsFalse("Active RM services are started", rm.AreActiveServicesRunning ()); CheckMonitorHealth(); }
public virtual void TestRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider () { configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); string[] defaultTestUserGroups = new string[] { "dummy_group1", "dummy_group2" }; UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("dummyUser", defaultTestUserGroups); string user = ugi.GetUserName(); IList <string> groupWithInit = new AList <string>(2); for (int i = 0; i < ugi.GetGroupNames().Length; i++) { groupWithInit.AddItem(ugi.GetGroupNames()[i]); } // upload default configurations UploadDefaultConfiguration(); Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestRMAdminService.MockUnixGroupsMapping ), typeof(GroupMappingServiceProvider)); UploadConfiguration(conf, "core-site.xml"); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } // Make sure RM will use the updated GroupMappingServiceProvider IList <string> groupBefore = new AList <string>(Groups.GetUserToGroupsMappingService (configuration).GetGroups(user)); NUnit.Framework.Assert.IsTrue(groupBefore.Contains("test_group_A") && groupBefore .Contains("test_group_B") && groupBefore.Contains("test_group_C") && groupBefore .Count == 3); NUnit.Framework.Assert.IsTrue(groupWithInit.Count != groupBefore.Count); NUnit.Framework.Assert.IsFalse(groupWithInit.Contains("test_group_A") || groupWithInit .Contains("test_group_B") || groupWithInit.Contains("test_group_C")); // update the groups TestRMAdminService.MockUnixGroupsMapping.UpdateGroups(); rm.adminService.RefreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest.NewInstance ()); IList <string> groupAfter = Groups.GetUserToGroupsMappingService(configuration).GetGroups (user); // should get the updated groups NUnit.Framework.Assert.IsTrue(groupAfter.Contains("test_group_D") && groupAfter.Contains ("test_group_E") && groupAfter.Contains("test_group_F") && groupAfter.Count == 3 ); }
public virtual void TestRefreshSuperUserGroupsWithFileSystemBasedConfigurationProvider () { configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); //upload default configurations UploadDefaultConfiguration(); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } Configuration coreConf = new Configuration(false); coreConf.Set("hadoop.proxyuser.test.groups", "test_groups"); coreConf.Set("hadoop.proxyuser.test.hosts", "test_hosts"); UploadConfiguration(coreConf, "core-site.xml"); rm.adminService.RefreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequest .NewInstance()); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups ()["hadoop.proxyuser.test.groups"].Count == 1); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups ()["hadoop.proxyuser.test.groups"].Contains("test_groups")); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts ()["hadoop.proxyuser.test.hosts"].Count == 1); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts ()["hadoop.proxyuser.test.hosts"].Contains("test_hosts")); Configuration yarnConf = new Configuration(false); yarnConf.Set("yarn.resourcemanager.proxyuser.test.groups", "test_groups_1"); yarnConf.Set("yarn.resourcemanager.proxyuser.test.hosts", "test_hosts_1"); UploadConfiguration(yarnConf, "yarn-site.xml"); // RM specific configs will overwrite the common ones rm.adminService.RefreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequest .NewInstance()); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups ()["hadoop.proxyuser.test.groups"].Count == 1); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups ()["hadoop.proxyuser.test.groups"].Contains("test_groups_1")); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts ()["hadoop.proxyuser.test.hosts"].Count == 1); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts ()["hadoop.proxyuser.test.hosts"].Contains("test_hosts_1")); }
public virtual void TestRefreshNodesWithLocalConfigurationProvider() { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); try { rm.adminService.RefreshNodes(RefreshNodesRequest.NewInstance()); } catch (Exception) { NUnit.Framework.Assert.Fail("Using localConfigurationProvider. Should not get any exception." ); } }
/// <exception cref="System.IO.IOException"/> protected internal virtual void StartRMs(MockRM rm1, Configuration confForRM1, MockRM rm2, Configuration confForRM2) { rm1.Init(confForRM1); rm1.Start(); NUnit.Framework.Assert.IsTrue(rm1.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState .Standby); rm2.Init(confForRM2); rm2.Start(); NUnit.Framework.Assert.IsTrue(rm2.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState .Standby); rm1.adminService.TransitionToActive(requestInfo); NUnit.Framework.Assert.IsTrue(rm1.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState .Active); }
public virtual void TestRMDispatcherForHA() { string errorMessageForEventHandler = "Expect to get the same number of handlers"; string errorMessageForService = "Expect to get the same number of services"; configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); Configuration conf = new YarnConfiguration(configuration); rm = new _MockRM_313(conf); rm.Init(conf); int expectedEventHandlerCount = ((TestRMHA.MyCountingDispatcher)rm.GetRMContext() .GetDispatcher()).GetEventHandlerCount(); int expectedServiceCount = rm.GetServices().Count; NUnit.Framework.Assert.IsTrue(expectedEventHandlerCount != 0); HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing , rm.adminService.GetServiceStatus().GetState()); NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started" , rm.adminService.GetServiceStatus().IsReadyToBecomeActive()); rm.Start(); //call transitions to standby and active a couple of times rm.adminService.TransitionToStandby(requestInfo); rm.adminService.TransitionToActive(requestInfo); rm.adminService.TransitionToStandby(requestInfo); rm.adminService.TransitionToActive(requestInfo); rm.adminService.TransitionToStandby(requestInfo); TestRMHA.MyCountingDispatcher dispatcher = (TestRMHA.MyCountingDispatcher)rm.GetRMContext ().GetDispatcher(); NUnit.Framework.Assert.IsTrue(!dispatcher.IsStopped()); rm.adminService.TransitionToActive(requestInfo); NUnit.Framework.Assert.AreEqual(errorMessageForEventHandler, expectedEventHandlerCount , ((TestRMHA.MyCountingDispatcher)rm.GetRMContext().GetDispatcher()).GetEventHandlerCount ()); NUnit.Framework.Assert.AreEqual(errorMessageForService, expectedServiceCount, rm. GetServices().Count); // Keep the dispatcher reference before transitioning to standby dispatcher = (TestRMHA.MyCountingDispatcher)rm.GetRMContext().GetDispatcher(); rm.adminService.TransitionToStandby(requestInfo); NUnit.Framework.Assert.AreEqual(errorMessageForEventHandler, expectedEventHandlerCount , ((TestRMHA.MyCountingDispatcher)rm.GetRMContext().GetDispatcher()).GetEventHandlerCount ()); NUnit.Framework.Assert.AreEqual(errorMessageForService, expectedServiceCount, rm. GetServices().Count); NUnit.Framework.Assert.IsTrue(dispatcher.IsStopped()); rm.Stop(); }
public virtual void TestFailoverClearsRMContext() { configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); configuration.SetBoolean(YarnConfiguration.RecoveryEnabled, true); Configuration conf = new YarnConfiguration(configuration); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); // 1. start RM rm = new MockRM(conf, memStore); rm.Init(conf); rm.Start(); HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); CheckMonitorHealth(); CheckStandbyRMFunctionality(); // 2. Transition to active rm.adminService.TransitionToActive(requestInfo); CheckMonitorHealth(); CheckActiveRMFunctionality(); VerifyClusterMetrics(1, 1, 1, 1, 2048, 1); NUnit.Framework.Assert.AreEqual(1, rm.GetRMContext().GetRMNodes().Count); NUnit.Framework.Assert.AreEqual(1, rm.GetRMContext().GetRMApps().Count); // 3. Create new RM rm = new _MockRM_550(conf, memStore); rm.Init(conf); rm.Start(); CheckMonitorHealth(); CheckStandbyRMFunctionality(); // 4. Try Transition to active, throw exception try { rm.adminService.TransitionToActive(requestInfo); NUnit.Framework.Assert.Fail("Transitioned to Active should throw exception."); } catch (Exception e) { NUnit.Framework.Assert.IsTrue("Error when transitioning to Active mode".Contains( e.Message)); } // 5. Clears the metrics VerifyClusterMetrics(0, 0, 0, 0, 0, 0); NUnit.Framework.Assert.AreEqual(0, rm.GetRMContext().GetRMNodes().Count); NUnit.Framework.Assert.AreEqual(0, rm.GetRMContext().GetRMApps().Count); }
public virtual void TestRefreshNodesWithFileSystemBasedConfigurationProvider() { configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); // upload default configurations UploadDefaultConfiguration(); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } FilePath excludeHostsFile = new FilePath(tmpDir.ToString(), "excludeHosts"); if (excludeHostsFile.Exists()) { excludeHostsFile.Delete(); } if (!excludeHostsFile.CreateNewFile()) { NUnit.Framework.Assert.Fail("Can not create " + "excludeHosts"); } PrintWriter fileWriter = new PrintWriter(excludeHostsFile); fileWriter.Write("0.0.0.0:123"); fileWriter.Close(); UploadToRemoteFileSystem(new Path(excludeHostsFile.GetAbsolutePath())); Configuration yarnConf = new YarnConfiguration(); yarnConf.Set(YarnConfiguration.RmNodesExcludeFilePath, this.workingPath + "/excludeHosts" ); UploadConfiguration(yarnConf, YarnConfiguration.YarnSiteConfigurationFile); rm.adminService.RefreshNodes(RefreshNodesRequest.NewInstance()); ICollection <string> excludeHosts = rm.GetNodesListManager().GetHostsReader().GetExcludedHosts (); NUnit.Framework.Assert.IsTrue(excludeHosts.Count == 1); NUnit.Framework.Assert.IsTrue(excludeHosts.Contains("0.0.0.0:123")); }
public virtual void TestRefreshAclWithDaemonUser() { string daemonUser = UserGroupInformation.GetCurrentUser().GetShortUserName(); configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); UploadDefaultConfiguration(); YarnConfiguration yarnConf = new YarnConfiguration(); yarnConf.Set(YarnConfiguration.YarnAdminAcl, daemonUser + "xyz"); UploadConfiguration(yarnConf, "yarn-site.xml"); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } NUnit.Framework.Assert.AreEqual(daemonUser + "xyz," + daemonUser, rm.adminService .GetAccessControlList().GetAclString().Trim()); yarnConf = new YarnConfiguration(); yarnConf.Set(YarnConfiguration.YarnAdminAcl, daemonUser + "abc"); UploadConfiguration(yarnConf, "yarn-site.xml"); try { rm.adminService.RefreshAdminAcls(RefreshAdminAclsRequest.NewInstance()); } catch (YarnException e) { if (e.InnerException != null && e.InnerException is AccessControlException) { NUnit.Framework.Assert.Fail("Refresh should not have failed due to incorrect ACL" ); } throw; } NUnit.Framework.Assert.AreEqual(daemonUser + "abc," + daemonUser, rm.adminService .GetAccessControlList().GetAclString().Trim()); }
public virtual void TestAdminRefreshQueuesWithLocalConfigurationProvider() { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); CapacityScheduler cs = (CapacityScheduler)rm.GetRMContext().GetScheduler(); int maxAppsBefore = cs.GetConfiguration().GetMaximumSystemApplications(); try { rm.adminService.RefreshQueues(RefreshQueuesRequest.NewInstance()); NUnit.Framework.Assert.AreEqual(maxAppsBefore, cs.GetConfiguration().GetMaximumSystemApplications ()); } catch (Exception) { NUnit.Framework.Assert.Fail("Using localConfigurationProvider. Should not get any exception." ); } }
public virtual void TestRMStartsWithoutConfigurationFilesProvided() { // enable FileSystemBasedConfigurationProvider without uploading // any configuration files into Remote File System. configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); // The configurationProvider will return NULL instead of // throwing out Exceptions, if there are no configuration files provided. // RM will not load the remote Configuration files, // and should start successfully. try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } }
/// <exception cref="System.Exception"/> public virtual void TestTransitionedToActiveRefreshFail() { configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); YarnConfiguration conf = new YarnConfiguration(configuration); configuration = new CapacitySchedulerConfiguration(conf); rm = new _MockRM_588(this, configuration); rm.Init(configuration); rm.Start(); HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); configuration.Set("yarn.scheduler.capacity.root.default.capacity", "100"); rm.adminService.TransitionToStandby(requestInfo); NUnit.Framework.Assert.AreEqual(HAServiceProtocol.HAServiceState.Standby, rm.GetRMContext ().GetHAServiceState()); configuration.Set("yarn.scheduler.capacity.root.default.capacity", "200"); try { rm.adminService.TransitionToActive(requestInfo); } catch (Exception e) { NUnit.Framework.Assert.IsTrue("Error on refreshAll during transistion to Active". Contains(e.Message)); } TestRMHA.FailFastDispatcher dispatcher = ((TestRMHA.FailFastDispatcher)rm.rmContext .GetDispatcher()); dispatcher.Await(); NUnit.Framework.Assert.AreEqual(1, dispatcher.GetEventCount()); // Making correct conf and check the state configuration.Set("yarn.scheduler.capacity.root.default.capacity", "100"); rm.adminService.TransitionToActive(requestInfo); NUnit.Framework.Assert.AreEqual(HAServiceProtocol.HAServiceState.Active, rm.GetRMContext ().GetHAServiceState()); rm.adminService.TransitionToStandby(requestInfo); NUnit.Framework.Assert.AreEqual(HAServiceProtocol.HAServiceState.Standby, rm.GetRMContext ().GetHAServiceState()); }
/// <exception cref="System.Exception"/> public virtual void TestFailoverWhenTransitionToActiveThrowException() { configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); Configuration conf = new YarnConfiguration(configuration); MemoryRMStateStore memStore = new _MemoryRMStateStore_414(); // first time throw exception // start RM memStore.Init(conf); rm = new MockRM(conf, memStore); rm.Init(conf); HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing , rm.adminService.GetServiceStatus().GetState()); NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started" , rm.adminService.GetServiceStatus().IsReadyToBecomeActive()); CheckMonitorHealth(); rm.Start(); CheckMonitorHealth(); CheckStandbyRMFunctionality(); // 2. Try Transition to active, throw exception try { rm.adminService.TransitionToActive(requestInfo); NUnit.Framework.Assert.Fail("Transitioned to Active should throw exception."); } catch (Exception e) { NUnit.Framework.Assert.IsTrue("Error when transitioning to Active mode".Contains( e.Message)); } // 3. Transition to active, success rm.adminService.TransitionToActive(requestInfo); CheckMonitorHealth(); CheckActiveRMFunctionality(); }
public virtual void InnerTestHAWithRMHostName(bool includeBindHost) { //this is run two times, with and without a bind host configured if (includeBindHost) { configuration.Set(YarnConfiguration.RmBindHost, "9.9.9.9"); } //test if both RM_HOSTBANE_{rm_id} and RM_RPCADDRESS_{rm_id} are set //We should only read rpc addresses from RM_RPCADDRESS_{rm_id} configuration configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm1NodeId), "1.1.1.1" ); configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm2NodeId), "0.0.0.0" ); configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm3NodeId), "2.2.2.2" ); try { Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); foreach (string confKey in YarnConfiguration.GetServiceAddressConfKeys(conf)) { NUnit.Framework.Assert.AreEqual("RPC address not set for " + confKey, Rm1Address, conf.Get(HAUtil.AddSuffix(confKey, Rm1NodeId))); NUnit.Framework.Assert.AreEqual("RPC address not set for " + confKey, Rm2Address, conf.Get(HAUtil.AddSuffix(confKey, Rm2NodeId))); NUnit.Framework.Assert.AreEqual("RPC address not set for " + confKey, Rm3Address, conf.Get(HAUtil.AddSuffix(confKey, Rm3NodeId))); if (includeBindHost) { NUnit.Framework.Assert.AreEqual("Web address misconfigured WITH bind-host", Sharpen.Runtime.Substring (rm.webAppAddress, 0, 7), "9.9.9.9"); } else { //YarnConfiguration tries to figure out which rm host it's on by binding to it, //which doesn't happen for any of these fake addresses, so we end up with 0.0.0.0 NUnit.Framework.Assert.AreEqual("Web address misconfigured WITHOUT bind-host", Sharpen.Runtime.Substring (rm.webAppAddress, 0, 7), "0.0.0.0"); } } } catch (YarnRuntimeException) { NUnit.Framework.Assert.Fail("Should not throw any exceptions."); } //test if only RM_HOSTBANE_{rm_id} is set configuration.Clear(); configuration.SetBoolean(YarnConfiguration.RmHaEnabled, true); configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm2NodeId); configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm1NodeId), "1.1.1.1" ); configuration.Set(HAUtil.AddSuffix(YarnConfiguration.RmHostname, Rm2NodeId), "0.0.0.0" ); try { Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.AreEqual("RPC address not set for " + YarnConfiguration.RmAddress , "1.1.1.1:8032", conf.Get(HAUtil.AddSuffix(YarnConfiguration.RmAddress, Rm1NodeId ))); NUnit.Framework.Assert.AreEqual("RPC address not set for " + YarnConfiguration.RmAddress , "0.0.0.0:8032", conf.Get(HAUtil.AddSuffix(YarnConfiguration.RmAddress, Rm2NodeId ))); } catch (YarnRuntimeException) { NUnit.Framework.Assert.Fail("Should not throw any exceptions."); } }
public virtual void TestRMHAWithFileSystemBasedConfiguration() { HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); configuration.SetBoolean(YarnConfiguration.RmHaEnabled, true); configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); configuration.Set(YarnConfiguration.RmHaIds, "rm1,rm2"); int @base = 100; foreach (string confKey in YarnConfiguration.GetServiceAddressConfKeys(configuration )) { configuration.Set(HAUtil.AddSuffix(confKey, "rm1"), "0.0.0.0:" + (@base + 20)); configuration.Set(HAUtil.AddSuffix(confKey, "rm2"), "0.0.0.0:" + (@base + 40)); @base = @base * 2; } Configuration conf1 = new Configuration(configuration); conf1.Set(YarnConfiguration.RmHaId, "rm1"); Configuration conf2 = new Configuration(configuration); conf2.Set(YarnConfiguration.RmHaId, "rm2"); // upload default configurations UploadDefaultConfiguration(); MockRM rm1 = null; MockRM rm2 = null; try { rm1 = new MockRM(conf1); rm1.Init(conf1); rm1.Start(); NUnit.Framework.Assert.IsTrue(rm1.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState .Standby); rm2 = new MockRM(conf2); rm2.Init(conf1); rm2.Start(); NUnit.Framework.Assert.IsTrue(rm2.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState .Standby); rm1.adminService.TransitionToActive(requestInfo); NUnit.Framework.Assert.IsTrue(rm1.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState .Active); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.Set("yarn.scheduler.capacity.maximum-applications", "5000"); UploadConfiguration(csConf, "capacity-scheduler.xml"); rm1.adminService.RefreshQueues(RefreshQueuesRequest.NewInstance()); int maxApps = ((CapacityScheduler)rm1.GetRMContext().GetScheduler()).GetConfiguration ().GetMaximumSystemApplications(); NUnit.Framework.Assert.AreEqual(maxApps, 5000); // Before failover happens, the maxApps is // still the default value on the standby rm : rm2 int maxAppsBeforeFailOver = ((CapacityScheduler)rm2.GetRMContext().GetScheduler() ).GetConfiguration().GetMaximumSystemApplications(); NUnit.Framework.Assert.AreEqual(maxAppsBeforeFailOver, 10000); // Do the failover rm1.adminService.TransitionToStandby(requestInfo); rm2.adminService.TransitionToActive(requestInfo); NUnit.Framework.Assert.IsTrue(rm1.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState .Standby); NUnit.Framework.Assert.IsTrue(rm2.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState .Active); int maxAppsAfter = ((CapacityScheduler)rm2.GetRMContext().GetScheduler()).GetConfiguration ().GetMaximumSystemApplications(); NUnit.Framework.Assert.AreEqual(maxAppsAfter, 5000); } finally { if (rm1 != null) { rm1.Stop(); } if (rm2 != null) { rm2.Stop(); } } }
public virtual void TestTransitionsWhenAutomaticFailoverEnabled() { string ErrUnforcedRequest = "User request succeeded even when " + "automatic failover is enabled"; Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); rm.Start(); HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); // Transition to standby try { rm.adminService.TransitionToStandby(requestInfo); NUnit.Framework.Assert.Fail(ErrUnforcedRequest); } catch (AccessControlException) { } // expected CheckMonitorHealth(); CheckStandbyRMFunctionality(); // Transition to active try { rm.adminService.TransitionToActive(requestInfo); NUnit.Framework.Assert.Fail(ErrUnforcedRequest); } catch (AccessControlException) { } // expected CheckMonitorHealth(); CheckStandbyRMFunctionality(); string ErrForcedRequest = "Forced request by user should work " + "even if automatic failover is enabled"; requestInfo = new HAServiceProtocol.StateChangeRequestInfo(HAServiceProtocol.RequestSource .RequestByUserForced); // Transition to standby try { rm.adminService.TransitionToStandby(requestInfo); } catch (AccessControlException) { NUnit.Framework.Assert.Fail(ErrForcedRequest); } CheckMonitorHealth(); CheckStandbyRMFunctionality(); // Transition to active try { rm.adminService.TransitionToActive(requestInfo); } catch (AccessControlException) { NUnit.Framework.Assert.Fail(ErrForcedRequest); } CheckMonitorHealth(); CheckActiveRMFunctionality(); }