public virtual void TestAdminAclsWithFileSystemBasedConfigurationProvider() { configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); //upload default configurations UploadDefaultConfiguration(); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } string aclStringBefore = rm.adminService.GetAccessControlList().GetAclString().Trim (); YarnConfiguration yarnConf = new YarnConfiguration(); yarnConf.Set(YarnConfiguration.YarnAdminAcl, "world:anyone:rwcda"); UploadConfiguration(yarnConf, "yarn-site.xml"); rm.adminService.RefreshAdminAcls(RefreshAdminAclsRequest.NewInstance()); string aclStringAfter = rm.adminService.GetAccessControlList().GetAclString().Trim (); NUnit.Framework.Assert.IsTrue(!aclStringAfter.Equals(aclStringBefore)); NUnit.Framework.Assert.AreEqual(aclStringAfter, "world:anyone:rwcda," + UserGroupInformation .GetCurrentUser().GetShortUserName()); }
public virtual void TestNodeUpdateBeforeAppAttemptInit() { FifoScheduler scheduler = new FifoScheduler(); MockRM rm = new MockRM(conf); scheduler.SetRMContext(rm.GetRMContext()); scheduler.Init(conf); scheduler.Start(); scheduler.Reinitialize(conf, rm.GetRMContext()); RMNode node = MockNodes.NewNodeInfo(1, Resources.CreateResource(1024, 4), 1, "127.0.0.1" ); scheduler.Handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = ApplicationId.NewInstance(0, 1); scheduler.AddApplication(appId, "queue1", "user1", false); NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node); try { scheduler.Handle(updateEvent); } catch (ArgumentNullException) { NUnit.Framework.Assert.Fail(); } ApplicationAttemptId attId = ApplicationAttemptId.NewInstance(appId, 1); scheduler.AddApplicationAttempt(attId, false, false); rm.Stop(); }
/// <exception cref="System.Exception"/> private void TestMinimumAllocation(YarnConfiguration conf, int testAlloc) { MockRM rm = new MockRM(conf); rm.Start(); // Register node1 MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb); // Submit an application RMApp app1 = rm.SubmitApp(testAlloc); // kick the scheduling nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId ()); int checkAlloc = conf.GetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, YarnConfiguration .DefaultRmSchedulerMinimumAllocationMb); NUnit.Framework.Assert.AreEqual(checkAlloc, report_nm1.GetUsedResource().GetMemory ()); rm.Stop(); }
public virtual void TestNodeRegistrationVersionLessThanRM() { WriteToHostsFile("host2"); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath()); conf.Set(YarnConfiguration.RmNodemanagerMinimumVersion, "EqualToRM"); rm = new MockRM(conf); rm.Start(); string nmVersion = "1.9.9"; ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService(); RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest >(); NodeId nodeId = NodeId.NewInstance("host2", 1234); Resource capability = BuilderUtils.NewResource(1024, 1); req.SetResource(capability); req.SetNodeId(nodeId); req.SetHttpPort(1234); req.SetNMVersion(nmVersion); // trying to register a invalid node. RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager (req); NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response.GetNodeAction()); NUnit.Framework.Assert.IsTrue("Diagnostic message did not contain: 'Disallowed NodeManager " + "Version " + nmVersion + ", is less than the minimum version'", response.GetDiagnosticsMessage ().Contains("Disallowed NodeManager Version " + nmVersion + ", is less than the minimum version " )); }
public static void Setup() { RMStateStore store = RMStateStoreFactory.GetStore(conf); conf.SetBoolean(YarnConfiguration.YarnAclEnable, true); AccessControlList adminACL = new AccessControlList(string.Empty); adminACL.AddGroup(SuperGroup); conf.Set(YarnConfiguration.YarnAdminAcl, adminACL.GetAclString()); resourceManager = new _MockRM_105(conf); new _Thread_127().Start(); int waitCount = 0; while (resourceManager.GetServiceState() == Service.STATE.Inited && waitCount++ < 60) { Log.Info("Waiting for RM to start..."); Sharpen.Thread.Sleep(1500); } if (resourceManager.GetServiceState() != Service.STATE.Started) { // RM could have failed. throw new IOException("ResourceManager failed to start. Final state is " + resourceManager .GetServiceState()); } UserGroupInformation owner = UserGroupInformation.CreateRemoteUser(AppOwner); rmClient = owner.DoAs(new _PrivilegedExceptionAction_152()); }
/// <exception cref="System.Exception"/> public virtual void TestInvalidatedAMHostPortOnAMRestart() { MockRM rm1 = new MockRM(conf); rm1.Start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService()); nm1.RegisterNode(); // a failed app RMApp app2 = rm1.SubmitApp(200); MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1); nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete); am2.WaitForState(RMAppAttemptState.Failed); rm1.WaitForState(app2.GetApplicationId(), RMAppState.Accepted); // before new attempt is launched, the app report returns the invalid AM // host and port. GetApplicationReportRequest request1 = GetApplicationReportRequest.NewInstance(app2 .GetApplicationId()); ApplicationReport report1 = rm1.GetClientRMService().GetApplicationReport(request1 ).GetApplicationReport(); NUnit.Framework.Assert.AreEqual("N/A", report1.GetHost()); NUnit.Framework.Assert.AreEqual(-1, report1.GetRpcPort()); }
public virtual void TestHandleContainerStatusInvalidCompletions() { rm = new MockRM(new YarnConfiguration()); rm.Start(); EventHandler handler = Org.Mockito.Mockito.Spy(rm.GetRMContext().GetDispatcher(). GetEventHandler()); // Case 1: Unmanaged AM RMApp app = rm.SubmitApp(1024, true); // Case 1.1: AppAttemptId is null NMContainerStatus report = NMContainerStatus.NewInstance(ContainerId.NewContainerId (ApplicationAttemptId.NewInstance(app.GetApplicationId(), 2), 1), ContainerState .Complete, Resource.NewInstance(1024, 1), "Dummy Completed", 0, Priority.NewInstance (10), 1234); rm.GetResourceTrackerService().HandleNMContainerStatus(report, null); Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event )Matchers.Any()); // Case 1.2: Master container is null RMAppAttemptImpl currentAttempt = (RMAppAttemptImpl)app.GetCurrentAppAttempt(); currentAttempt.SetMasterContainer(null); report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(currentAttempt. GetAppAttemptId(), 0), ContainerState.Complete, Resource.NewInstance(1024, 1), "Dummy Completed" , 0, Priority.NewInstance(10), 1234); rm.GetResourceTrackerService().HandleNMContainerStatus(report, null); Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event )Matchers.Any()); // Case 2: Managed AM app = rm.SubmitApp(1024); // Case 2.1: AppAttemptId is null report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(ApplicationAttemptId .NewInstance(app.GetApplicationId(), 2), 1), ContainerState.Complete, Resource.NewInstance (1024, 1), "Dummy Completed", 0, Priority.NewInstance(10), 1234); try { rm.GetResourceTrackerService().HandleNMContainerStatus(report, null); } catch (Exception) { } // expected - ignore Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event )Matchers.Any()); // Case 2.2: Master container is null currentAttempt = (RMAppAttemptImpl)app.GetCurrentAppAttempt(); currentAttempt.SetMasterContainer(null); report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(currentAttempt. GetAppAttemptId(), 0), ContainerState.Complete, Resource.NewInstance(1024, 1), "Dummy Completed" , 0, Priority.NewInstance(10), 1234); try { rm.GetResourceTrackerService().HandleNMContainerStatus(report, null); } catch (Exception) { } // expected - ignore Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event )Matchers.Any()); }
public virtual void Setup() { conf = CreateConfiguration(); rpc = YarnRPC.Create(conf); rmAddress = conf.GetSocketAddr(YarnConfiguration.RmAddress, YarnConfiguration.DefaultRmAddress , YarnConfiguration.DefaultRmPort); AccessControlList adminACL = new AccessControlList(string.Empty); conf.Set(YarnConfiguration.YarnAdminAcl, adminACL.GetAclString()); resourceManager = new _MockRM_85(conf); new _Thread_96(this).Start(); int waitCount = 0; while (resourceManager.GetServiceState() == Service.STATE.Inited && waitCount++ < 60) { Log.Info("Waiting for RM to start..."); Sharpen.Thread.Sleep(1500); } if (resourceManager.GetServiceState() != Service.STATE.Started) { // RM could have failed. throw new IOException("ResourceManager failed to start. Final state is " + resourceManager .GetServiceState()); } }
/// <exception cref="System.Exception"/> public virtual void TestTransitionedToStandbyShouldNotHang() { configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); Configuration conf = new YarnConfiguration(configuration); MemoryRMStateStore memStore = new _MemoryRMStateStore_464(); memStore.Init(conf); rm = new _MockRM_472(conf, memStore); rm.Init(conf); HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing , rm.adminService.GetServiceStatus().GetState()); NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started" , rm.adminService.GetServiceStatus().IsReadyToBecomeActive()); CheckMonitorHealth(); rm.Start(); CheckMonitorHealth(); CheckStandbyRMFunctionality(); // 2. Transition to Active. rm.adminService.TransitionToActive(requestInfo); // 3. Try Transition to standby Sharpen.Thread t = new Sharpen.Thread(new _Runnable_498(this)); // TODO Auto-generated catch block t.Start(); rm.GetRMContext().GetStateStore().UpdateApplicationState(null); t.Join(); // wait for thread to finish rm.adminService.TransitionToStandby(requestInfo); CheckStandbyRMFunctionality(); rm.Stop(); }
public virtual void TestHAIDLookup() { //test implicitly lookup HA-ID Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.AreEqual(conf.Get(YarnConfiguration.RmHaId), Rm2NodeId); //test explicitly lookup HA-ID configuration.Set(YarnConfiguration.RmHaId, Rm1NodeId); conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.AreEqual(conf.Get(YarnConfiguration.RmHaId), Rm1NodeId); //test if RM_HA_ID can not be found configuration.Set(YarnConfiguration.RmHaIds, Rm1NodeId + "," + Rm3NodeId); configuration.Unset(YarnConfiguration.RmHaId); conf = new YarnConfiguration(configuration); try { rm = new MockRM(conf); rm.Init(conf); NUnit.Framework.Assert.Fail("Should get an exception here."); } catch (Exception ex) { NUnit.Framework.Assert.IsTrue(ex.Message.Contains("Invalid configuration! Can not find valid RM_HA_ID." )); } }
public virtual void TestAdminRefreshQueuesWithFileSystemBasedConfigurationProvider () { configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); //upload default configurations UploadDefaultConfiguration(); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } CapacityScheduler cs = (CapacityScheduler)rm.GetRMContext().GetScheduler(); int maxAppsBefore = cs.GetConfiguration().GetMaximumSystemApplications(); CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.Set("yarn.scheduler.capacity.maximum-applications", "5000"); UploadConfiguration(csConf, "capacity-scheduler.xml"); rm.adminService.RefreshQueues(RefreshQueuesRequest.NewInstance()); int maxAppsAfter = cs.GetConfiguration().GetMaximumSystemApplications(); NUnit.Framework.Assert.AreEqual(maxAppsAfter, 5000); NUnit.Framework.Assert.IsTrue(maxAppsAfter != maxAppsBefore); }
public virtual void TestProxyUserConfiguration() { MockRM rm = null; try { rm = new MockRM(conf); rm.Start(); // wait for web server starting Sharpen.Thread.Sleep(10000); UserGroupInformation proxyUser = UserGroupInformation.CreateProxyUser(BarUser.GetShortUserName (), FooUser); try { ProxyUsers.GetDefaultImpersonationProvider().Authorize(proxyUser, ipAddress); } catch (AuthorizationException) { // Exception is not expected NUnit.Framework.Assert.Fail(); } } finally { if (rm != null) { rm.Stop(); rm.Close(); } } }
/// <exception cref="System.Exception"/> public virtual void TestAppCleanupWhenNMReconnects() { conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); // start RM MockRM rm1 = new MockRM(conf, memStore); rm1.Start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService()); nm1.RegisterNode(); // create app and launch the AM RMApp app0 = rm1.SubmitApp(200); MockAM am0 = LaunchAM(app0, rm1, nm1); nm1.NodeHeartbeat(am0.GetApplicationAttemptId(), 1, ContainerState.Complete); rm1.WaitForState(app0.GetApplicationId(), RMAppState.Failed); // wait for application cleanup message received WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId()); // reconnect NM with application still active nm1.RegisterNode(Arrays.AsList(app0.GetApplicationId())); WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId()); rm1.Stop(); }
public virtual void TestAddNewExcludePathToConfiguration() { Configuration conf = new Configuration(); rm = new MockRM(conf); rm.Start(); MockNM nm1 = rm.RegisterNode("host1:1234", 5120); MockNM nm2 = rm.RegisterNode("host2:5678", 10240); ClusterMetrics metrics = ClusterMetrics.GetMetrics(); System.Diagnostics.Debug.Assert((metrics != null)); int initialMetricCount = metrics.GetNumDecommisionedNMs(); NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual(NodeAction.Normal, nodeHeartbeat.GetNodeAction()); nodeHeartbeat = nm2.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual(NodeAction.Normal, nodeHeartbeat.GetNodeAction()); WriteToHostsFile("host2"); conf.Set(YarnConfiguration.RmNodesExcludeFilePath, hostFile.GetAbsolutePath()); rm.GetNodesListManager().RefreshNodes(conf); CheckDecommissionedNMCount(rm, ++initialMetricCount); nodeHeartbeat = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual("Node should not have been decomissioned.", NodeAction .Normal, nodeHeartbeat.GetNodeAction()); nodeHeartbeat = nm2.NodeHeartbeat(true); NUnit.Framework.Assert.AreEqual("Node should have been decomissioned but is in state" + nodeHeartbeat.GetNodeAction(), NodeAction.Shutdown, nodeHeartbeat.GetNodeAction ()); }
/// <exception cref="System.Exception"/> private void VerifySubmitApp(MockRM rm, RMApp app, ApplicationId expectedAppId) { int maxWaittingTimes = 20; int count = 0; while (true) { YarnApplicationState state = rm.GetApplicationReport(app.GetApplicationId()).GetYarnApplicationState (); if (!state.Equals(YarnApplicationState.New) && !state.Equals(YarnApplicationState .NewSaving)) { break; } if (count > maxWaittingTimes) { break; } Sharpen.Thread.Sleep(200); count++; } // Verify submittion is successful YarnApplicationState state_1 = rm.GetApplicationReport(app.GetApplicationId()).GetYarnApplicationState (); NUnit.Framework.Assert.IsTrue(state_1 == YarnApplicationState.Accepted || state_1 == YarnApplicationState.Submitted); NUnit.Framework.Assert.AreEqual(expectedAppId, app.GetApplicationId()); }
/// <exception cref="System.IO.IOException"/> protected internal virtual void StartRMsWithCustomizedRMAppManager() { Configuration conf1 = new Configuration(confForRM1); rm1 = new _MockRM_118(conf1, conf1); rm2 = new MockRM(confForRM2); StartRMs(rm1, conf1, rm2, confForRM2); }
/// <exception cref="System.IO.IOException"/> private void StartRMsWithCustomizedClientRMService() { Configuration conf1 = new Configuration(confForRM1); rm1 = new _MockRM_193(conf1); rm2 = new MockRM(confForRM2); StartRMs(rm1, conf1, rm2, confForRM2); }
/// <exception cref="System.Exception"/> public virtual void TestAppOnMultiNode() { Logger rootLogger = LogManager.GetRootLogger(); rootLogger.SetLevel(Level.Debug); conf.Set("yarn.scheduler.capacity.node-locality-delay", "-1"); MockRM rm = new MockRM(conf); rm.Start(); MockNM nm1 = rm.RegisterNode("h1:1234", 5120); MockNM nm2 = rm.RegisterNode("h2:5678", 10240); RMApp app = rm.SubmitApp(2000); //kick the scheduling nm1.NodeHeartbeat(true); RMAppAttempt attempt = app.GetCurrentAppAttempt(); MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId()); am.RegisterAppAttempt(); //request for containers int request = 13; am.Allocate("h1", 1000, request, new AList <ContainerId>()); //kick the scheduler IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId >()).GetAllocatedContainers(); int contReceived = conts.Count; while (contReceived < 3) { //only 3 containers are available on node1 nm1.NodeHeartbeat(true); Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers()); contReceived = conts.Count; Log.Info("Got " + contReceived + " containers. Waiting to get " + 3); Sharpen.Thread.Sleep(WaitSleepMs); } NUnit.Framework.Assert.AreEqual(3, conts.Count); //send node2 heartbeat conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers (); contReceived = conts.Count; while (contReceived < 10) { nm2.NodeHeartbeat(true); Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers()); contReceived = conts.Count; Log.Info("Got " + contReceived + " containers. Waiting to get " + 10); Sharpen.Thread.Sleep(WaitSleepMs); } NUnit.Framework.Assert.AreEqual(10, conts.Count); am.UnregisterAppAttempt(); nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete); am.WaitForState(RMAppAttemptState.Finished); rm.Stop(); }
/// <exception cref="System.Exception"/> public virtual void TestInvalidContainerReleaseRequest() { MockRM rm = new MockRM(conf); try { rm.Start(); // Register node1 MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb); // Submit an application RMApp app1 = rm.SubmitApp(1024); // kick the scheduling nm1.NodeHeartbeat(true); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId()); am1.RegisterAppAttempt(); am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1); AllocateResponse alloc1Response = am1.Schedule(); // send the request // kick the scheduler nm1.NodeHeartbeat(true); while (alloc1Response.GetAllocatedContainers().Count < 1) { Log.Info("Waiting for containers to be created for app 1..."); Sharpen.Thread.Sleep(1000); alloc1Response = am1.Schedule(); } NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0); RMApp app2 = rm.SubmitApp(1024); nm1.NodeHeartbeat(true); RMAppAttempt attempt2 = app2.GetCurrentAppAttempt(); MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId()); am2.RegisterAppAttempt(); // Now trying to release container allocated for app1 -> appAttempt1. ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId(); am2.AddContainerToBeReleased(cId); try { am2.Schedule(); NUnit.Framework.Assert.Fail("Exception was expected!!"); } catch (InvalidContainerReleaseException e) { StringBuilder sb = new StringBuilder("Cannot release container : "); sb.Append(cId.ToString()); sb.Append(" not belonging to this application attempt : "); sb.Append(attempt2.GetAppAttemptId().ToString()); NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString())); } } finally { if (rm != null) { rm.Stop(); } } }
/// <exception cref="System.Exception"/> private MockAM LaunchAM(RMApp app, MockRM rm, MockNM nm) { RMAppAttempt attempt = app.GetCurrentAppAttempt(); nm.NodeHeartbeat(true); MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId()); am.RegisterAppAttempt(); rm.WaitForState(app.GetApplicationId(), RMAppState.Running); return(am); }
public virtual void TestRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider () { configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); string[] defaultTestUserGroups = new string[] { "dummy_group1", "dummy_group2" }; UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("dummyUser", defaultTestUserGroups); string user = ugi.GetUserName(); IList <string> groupWithInit = new AList <string>(2); for (int i = 0; i < ugi.GetGroupNames().Length; i++) { groupWithInit.AddItem(ugi.GetGroupNames()[i]); } // upload default configurations UploadDefaultConfiguration(); Configuration conf = new Configuration(); conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestRMAdminService.MockUnixGroupsMapping ), typeof(GroupMappingServiceProvider)); UploadConfiguration(conf, "core-site.xml"); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } // Make sure RM will use the updated GroupMappingServiceProvider IList <string> groupBefore = new AList <string>(Groups.GetUserToGroupsMappingService (configuration).GetGroups(user)); NUnit.Framework.Assert.IsTrue(groupBefore.Contains("test_group_A") && groupBefore .Contains("test_group_B") && groupBefore.Contains("test_group_C") && groupBefore .Count == 3); NUnit.Framework.Assert.IsTrue(groupWithInit.Count != groupBefore.Count); NUnit.Framework.Assert.IsFalse(groupWithInit.Contains("test_group_A") || groupWithInit .Contains("test_group_B") || groupWithInit.Contains("test_group_C")); // update the groups TestRMAdminService.MockUnixGroupsMapping.UpdateGroups(); rm.adminService.RefreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest.NewInstance ()); IList <string> groupAfter = Groups.GetUserToGroupsMappingService(configuration).GetGroups (user); // should get the updated groups NUnit.Framework.Assert.IsTrue(groupAfter.Contains("test_group_D") && groupAfter.Contains ("test_group_E") && groupAfter.Contains("test_group_F") && groupAfter.Count == 3 ); }
// Other stuff is verified in the regular web-services related tests /// <summary>Test to verify the following RM HA transitions to the following states.</summary> /// <remarks> /// Test to verify the following RM HA transitions to the following states. /// 1. Standby: Should be a no-op /// 2. Active: Active services should start /// 3. Active: Should be a no-op. /// While active, submit a couple of jobs /// 4. Standby: Active services should stop /// 5. Active: Active services should start /// 6. Stop the RM: All services should stop and RM should not be ready to /// become Active /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestFailoverAndTransitions() { configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); Configuration conf = new YarnConfiguration(configuration); rm = new MockRM(conf); rm.Init(conf); HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing , rm.adminService.GetServiceStatus().GetState()); NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started" , rm.adminService.GetServiceStatus().IsReadyToBecomeActive()); CheckMonitorHealth(); rm.Start(); CheckMonitorHealth(); CheckStandbyRMFunctionality(); VerifyClusterMetrics(0, 0, 0, 0, 0, 0); // 1. Transition to Standby - must be a no-op rm.adminService.TransitionToStandby(requestInfo); CheckMonitorHealth(); CheckStandbyRMFunctionality(); VerifyClusterMetrics(0, 0, 0, 0, 0, 0); // 2. Transition to active rm.adminService.TransitionToActive(requestInfo); CheckMonitorHealth(); CheckActiveRMFunctionality(); VerifyClusterMetrics(1, 1, 1, 1, 2048, 1); // 3. Transition to active - no-op rm.adminService.TransitionToActive(requestInfo); CheckMonitorHealth(); CheckActiveRMFunctionality(); VerifyClusterMetrics(1, 2, 2, 2, 2048, 2); // 4. Transition to standby rm.adminService.TransitionToStandby(requestInfo); CheckMonitorHealth(); CheckStandbyRMFunctionality(); VerifyClusterMetrics(0, 0, 0, 0, 0, 0); // 5. Transition to active to check Active->Standby->Active works rm.adminService.TransitionToActive(requestInfo); CheckMonitorHealth(); CheckActiveRMFunctionality(); VerifyClusterMetrics(1, 1, 1, 1, 2048, 1); // 6. Stop the RM. All services should stop and RM should not be ready to // become active rm.Stop(); NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Stopping , rm.adminService.GetServiceStatus().GetState()); NUnit.Framework.Assert.IsFalse("RM is ready to become active even after it is stopped" , rm.adminService.GetServiceStatus().IsReadyToBecomeActive()); NUnit.Framework.Assert.IsFalse("Active RM services are started", rm.AreActiveServicesRunning ()); CheckMonitorHealth(); }
public virtual void TestAuthorizedAccess() { TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager (); rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager); rm.Start(); MockNM nm1 = rm.RegisterNode("localhost:1234", 5120); IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType , string>(2); acls[ApplicationAccessType.ViewApp] = "*"; RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls); nm1.NodeHeartbeat(true); int waitCount = 0; while (containerManager.containerTokens == null && waitCount++ < 20) { Log.Info("Waiting for AM Launch to happen.."); Sharpen.Thread.Sleep(1000); } NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens); RMAppAttempt attempt = app.GetCurrentAppAttempt(); ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId(); WaitForLaunchedState(attempt); // Create a client to the RM. Configuration conf = rm.GetConfig(); YarnRPC rpc = YarnRPC.Create(conf); UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId .ToString()); Credentials credentials = containerManager.GetContainerCredentials(); IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress(); Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS .SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens()); currentUser.AddToken(amRMToken); ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this , rpc, conf)); RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterApplicationMasterRequest>(); RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request ); NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey()); if (UserGroupInformation.IsSecurityEnabled()) { NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array ()).Length > 0); } NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs ()[ApplicationAccessType.ViewApp]); }
public virtual void TestDecommissionWithExcludeHosts() { Configuration conf = new Configuration(); conf.Set(YarnConfiguration.RmNodesExcludeFilePath, hostFile.GetAbsolutePath()); WriteToHostsFile(string.Empty); DrainDispatcher dispatcher = new DrainDispatcher(); rm = new _MockRM_162(dispatcher, conf); rm.Start(); MockNM nm1 = rm.RegisterNode("host1:1234", 5120); MockNM nm2 = rm.RegisterNode("host2:5678", 10240); MockNM nm3 = rm.RegisterNode("localhost:4433", 1024); dispatcher.Await(); int metricCount = ClusterMetrics.GetMetrics().GetNumDecommisionedNMs(); NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); nodeHeartbeat = nm2.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); dispatcher.Await(); // To test that IPs also work string ip = NetUtils.NormalizeHostName("localhost"); WriteToHostsFile("host2", ip); rm.GetNodesListManager().RefreshNodes(conf); CheckDecommissionedNMCount(rm, metricCount + 2); nodeHeartbeat = nm1.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); nodeHeartbeat = nm2.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue("The decommisioned metrics are not updated", NodeAction .Shutdown.Equals(nodeHeartbeat.GetNodeAction())); nodeHeartbeat = nm3.NodeHeartbeat(true); NUnit.Framework.Assert.IsTrue("The decommisioned metrics are not updated", NodeAction .Shutdown.Equals(nodeHeartbeat.GetNodeAction())); dispatcher.Await(); WriteToHostsFile(string.Empty); rm.GetNodesListManager().RefreshNodes(conf); nm3 = rm.RegisterNode("localhost:4433", 1024); dispatcher.Await(); nodeHeartbeat = nm3.NodeHeartbeat(true); dispatcher.Await(); NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction ())); // decommissined node is 1 since 1 node is rejoined after updating exclude // file CheckDecommissionedNMCount(rm, metricCount + 1); }
// Test even if AM container is allocated with containerId not equal to 1, the // following allocate requests from AM should be able to retrieve the // corresponding NM Token. /// <exception cref="System.Exception"/> public virtual void TestNMTokenSentForNormalContainer() { conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName ()); MockRM rm = new MockRM(conf); rm.Start(); MockNM nm1 = rm.RegisterNode("h1:1234", 5120); RMApp app = rm.SubmitApp(2000); RMAppAttempt attempt = app.GetCurrentAppAttempt(); // Call getNewContainerId to increase container Id so that the AM container // Id doesn't equal to one. CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler(); cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId(); // kick the scheduling nm1.NodeHeartbeat(true); MockAM am = MockRM.LaunchAM(app, rm, nm1); // am container Id not equal to 1. NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId () != 1); // NMSecretManager doesn't record the node on which the am is allocated. NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent (attempt.GetAppAttemptId(), nm1.GetNodeId())); am.RegisterAppAttempt(); rm.WaitForState(app.GetApplicationId(), RMAppState.Running); int NumContainers = 1; IList <Container> containers = new AList <Container>(); // nmTokens keeps track of all the nmTokens issued in the allocate call. IList <NMToken> expectedNMTokens = new AList <NMToken>(); // am1 allocate 1 container on nm1. while (true) { AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList <ContainerId>()); nm1.NodeHeartbeat(true); Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers()); Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens()); if (containers.Count == NumContainers) { break; } Sharpen.Thread.Sleep(200); System.Console.Out.WriteLine("Waiting for container to be allocated."); } NodeId nodeId = expectedNMTokens[0].GetNodeId(); // NMToken is sent for the allocated container. NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId); }
// This is to test AM Host and rpc port are invalidated after the am attempt // is killed or failed, so that client doesn't get the wrong information. /// <exception cref="System.Exception"/> public virtual void TestInvalidateAMHostPortWhenAMFailedOrKilled() { conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); MockRM rm1 = new MockRM(conf); rm1.Start(); // a succeeded app RMApp app1 = rm1.SubmitApp(200); MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService()); nm1.RegisterNode(); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); MockRM.FinishAMAndVerifyAppState(app1, rm1, nm1, am1); // a failed app RMApp app2 = rm1.SubmitApp(200); MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1); nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete); am2.WaitForState(RMAppAttemptState.Failed); rm1.WaitForState(app2.GetApplicationId(), RMAppState.Failed); // a killed app RMApp app3 = rm1.SubmitApp(200); MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm1); rm1.KillApp(app3.GetApplicationId()); rm1.WaitForState(app3.GetApplicationId(), RMAppState.Killed); rm1.WaitForState(am3.GetApplicationAttemptId(), RMAppAttemptState.Killed); GetApplicationsRequest request1 = GetApplicationsRequest.NewInstance(EnumSet.Of(YarnApplicationState .Finished, YarnApplicationState.Killed, YarnApplicationState.Failed)); GetApplicationsResponse response1 = rm1.GetClientRMService().GetApplications(request1 ); IList <ApplicationReport> appList1 = response1.GetApplicationList(); NUnit.Framework.Assert.AreEqual(3, appList1.Count); foreach (ApplicationReport report in appList1) { // killed/failed apps host and rpc port are invalidated. if (report.GetApplicationId().Equals(app2.GetApplicationId()) || report.GetApplicationId ().Equals(app3.GetApplicationId())) { NUnit.Framework.Assert.AreEqual("N/A", report.GetHost()); NUnit.Framework.Assert.AreEqual(-1, report.GetRpcPort()); } // succeeded app's host and rpc port is not invalidated if (report.GetApplicationId().Equals(app1.GetApplicationId())) { NUnit.Framework.Assert.IsFalse(report.GetHost().Equals("N/A")); NUnit.Framework.Assert.IsTrue(report.GetRpcPort() != -1); } } }
public virtual void TestSetRMIdentifierInRegistration() { Configuration conf = new Configuration(); rm = new MockRM(conf); rm.Start(); MockNM nm = new MockNM("host1:1234", 5120, rm.GetResourceTrackerService()); RegisterNodeManagerResponse response = nm.RegisterNode(); // Verify the RMIdentifier is correctly set in RegisterNodeManagerResponse NUnit.Framework.Assert.AreEqual(ResourceManager.GetClusterTimeStamp(), response.GetRMIdentifier ()); }
/// <exception cref="System.Exception"/> public virtual void TestAppCleanupWhenRMRestartedBeforeAppFinished() { conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); // start RM MockRM rm1 = new MockRM(conf, memStore); rm1.Start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 1024, rm1.GetResourceTrackerService()); nm1.RegisterNode(); MockNM nm2 = new MockNM("127.0.0.1:5678", 1024, rm1.GetResourceTrackerService()); nm2.RegisterNode(); // create app and launch the AM RMApp app0 = rm1.SubmitApp(200); MockAM am0 = LaunchAM(app0, rm1, nm1); // alloc another container on nm2 AllocateResponse allocResponse = am0.Allocate(Arrays.AsList(ResourceRequest.NewInstance (Priority.NewInstance(1), "*", Resource.NewInstance(1024, 0), 1)), null); while (null == allocResponse.GetAllocatedContainers() || allocResponse.GetAllocatedContainers ().IsEmpty()) { nm2.NodeHeartbeat(true); allocResponse = am0.Allocate(null, null); Sharpen.Thread.Sleep(1000); } // start new RM MockRM rm2 = new MockRM(conf, memStore); rm2.Start(); // nm1/nm2 register to rm2, and do a heartbeat nm1.SetResourceTrackerService(rm2.GetResourceTrackerService()); nm1.RegisterNode(Arrays.AsList(NMContainerStatus.NewInstance(ContainerId.NewContainerId (am0.GetApplicationAttemptId(), 1), ContainerState.Complete, Resource.NewInstance (1024, 1), string.Empty, 0, Priority.NewInstance(0), 1234)), Arrays.AsList(app0. GetApplicationId())); nm2.SetResourceTrackerService(rm2.GetResourceTrackerService()); nm2.RegisterNode(Arrays.AsList(app0.GetApplicationId())); // assert app state has been saved. rm2.WaitForState(app0.GetApplicationId(), RMAppState.Failed); // wait for application cleanup message received on NM1 WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId()); // wait for application cleanup message received on NM2 WaitForAppCleanupMessageRecved(nm2, app0.GetApplicationId()); rm1.Stop(); rm2.Stop(); }
public virtual void TestRefreshSuperUserGroupsWithFileSystemBasedConfigurationProvider () { configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider" ); //upload default configurations UploadDefaultConfiguration(); try { rm = new MockRM(configuration); rm.Init(configuration); rm.Start(); } catch (Exception) { NUnit.Framework.Assert.Fail("Should not get any exceptions"); } Configuration coreConf = new Configuration(false); coreConf.Set("hadoop.proxyuser.test.groups", "test_groups"); coreConf.Set("hadoop.proxyuser.test.hosts", "test_hosts"); UploadConfiguration(coreConf, "core-site.xml"); rm.adminService.RefreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequest .NewInstance()); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups ()["hadoop.proxyuser.test.groups"].Count == 1); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups ()["hadoop.proxyuser.test.groups"].Contains("test_groups")); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts ()["hadoop.proxyuser.test.hosts"].Count == 1); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts ()["hadoop.proxyuser.test.hosts"].Contains("test_hosts")); Configuration yarnConf = new Configuration(false); yarnConf.Set("yarn.resourcemanager.proxyuser.test.groups", "test_groups_1"); yarnConf.Set("yarn.resourcemanager.proxyuser.test.hosts", "test_hosts_1"); UploadConfiguration(yarnConf, "yarn-site.xml"); // RM specific configs will overwrite the common ones rm.adminService.RefreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequest .NewInstance()); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups ()["hadoop.proxyuser.test.groups"].Count == 1); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups ()["hadoop.proxyuser.test.groups"].Contains("test_groups_1")); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts ()["hadoop.proxyuser.test.hosts"].Count == 1); NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts ()["hadoop.proxyuser.test.hosts"].Contains("test_hosts_1")); }
/// <exception cref="System.Exception"/> private void CheckRebootedNMCount(MockRM rm2, int count) { int waitCount = 0; while (ClusterMetrics.GetMetrics().GetNumRebootedNMs() != count && waitCount++ < 20) { lock (this) { Sharpen.Runtime.Wait(this, 100); } } NUnit.Framework.Assert.AreEqual("The rebooted metrics are not updated", count, ClusterMetrics .GetMetrics().GetNumRebootedNMs()); }