/// <exception cref="System.Exception"/>
private void TestMinimumAllocation(YarnConfiguration conf, int testAlloc)
{
MockRM rm = new MockRM(conf);
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(testAlloc);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
SchedulerNodeReport report_nm1 = rm.GetResourceScheduler().GetNodeReport(nm1.GetNodeId
());
int checkAlloc = conf.GetInt(YarnConfiguration.RmSchedulerMinimumAllocationMb, YarnConfiguration
.DefaultRmSchedulerMinimumAllocationMb);
NUnit.Framework.Assert.AreEqual(checkAlloc, report_nm1.GetUsedResource().GetMemory
());
rm.Stop();
}
public virtual void TestAdminAclsWithFileSystemBasedConfigurationProvider()
{
configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
);
//upload default configurations
UploadDefaultConfiguration();
try
{
rm = new MockRM(configuration);
rm.Init(configuration);
rm.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Should not get any exceptions");
}
string aclStringBefore = rm.adminService.GetAccessControlList().GetAclString().Trim
();
YarnConfiguration yarnConf = new YarnConfiguration();
yarnConf.Set(YarnConfiguration.YarnAdminAcl, "world:anyone:rwcda");
UploadConfiguration(yarnConf, "yarn-site.xml");
rm.adminService.RefreshAdminAcls(RefreshAdminAclsRequest.NewInstance());
string aclStringAfter = rm.adminService.GetAccessControlList().GetAclString().Trim
();
NUnit.Framework.Assert.IsTrue(!aclStringAfter.Equals(aclStringBefore));
NUnit.Framework.Assert.AreEqual(aclStringAfter, "world:anyone:rwcda," + UserGroupInformation
.GetCurrentUser().GetShortUserName());
}
public virtual void TestAddNewExcludePathToConfiguration()
{
Configuration conf = new Configuration();
rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
MockNM nm2 = rm.RegisterNode("host2:5678", 10240);
ClusterMetrics metrics = ClusterMetrics.GetMetrics();
System.Diagnostics.Debug.Assert((metrics != null));
int initialMetricCount = metrics.GetNumDecommisionedNMs();
NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual(NodeAction.Normal, nodeHeartbeat.GetNodeAction());
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual(NodeAction.Normal, nodeHeartbeat.GetNodeAction());
WriteToHostsFile("host2");
conf.Set(YarnConfiguration.RmNodesExcludeFilePath, hostFile.GetAbsolutePath());
rm.GetNodesListManager().RefreshNodes(conf);
CheckDecommissionedNMCount(rm, ++initialMetricCount);
nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual("Node should not have been decomissioned.", NodeAction
.Normal, nodeHeartbeat.GetNodeAction());
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual("Node should have been decomissioned but is in state"
+ nodeHeartbeat.GetNodeAction(), NodeAction.Shutdown, nodeHeartbeat.GetNodeAction
());
}
/// <exception cref="System.Exception"/>
public virtual void TestTransitionedToStandbyShouldNotHang()
{
configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
Configuration conf = new YarnConfiguration(configuration);
MemoryRMStateStore memStore = new _MemoryRMStateStore_464();
memStore.Init(conf);
rm = new _MockRM_472(conf, memStore);
rm.Init(conf);
HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo
(HAServiceProtocol.RequestSource.RequestByUser);
NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing
, rm.adminService.GetServiceStatus().GetState());
NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started"
, rm.adminService.GetServiceStatus().IsReadyToBecomeActive());
CheckMonitorHealth();
rm.Start();
CheckMonitorHealth();
CheckStandbyRMFunctionality();
// 2. Transition to Active.
rm.adminService.TransitionToActive(requestInfo);
// 3. Try Transition to standby
Sharpen.Thread t = new Sharpen.Thread(new _Runnable_498(this));
// TODO Auto-generated catch block
t.Start();
rm.GetRMContext().GetStateStore().UpdateApplicationState(null);
t.Join();
// wait for thread to finish
rm.adminService.TransitionToStandby(requestInfo);
CheckStandbyRMFunctionality();
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidatedAMHostPortOnAMRestart()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
// a failed app
RMApp app2 = rm1.SubmitApp(200);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app2.GetApplicationId(), RMAppState.Accepted);
// before new attempt is launched, the app report returns the invalid AM
// host and port.
GetApplicationReportRequest request1 = GetApplicationReportRequest.NewInstance(app2
.GetApplicationId());
ApplicationReport report1 = rm1.GetClientRMService().GetApplicationReport(request1
).GetApplicationReport();
NUnit.Framework.Assert.AreEqual("N/A", report1.GetHost());
NUnit.Framework.Assert.AreEqual(-1, report1.GetRpcPort());
}
public virtual void TestNodeRegistrationVersionLessThanRM()
{
WriteToHostsFile("host2");
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
conf.Set(YarnConfiguration.RmNodemanagerMinimumVersion, "EqualToRM");
rm = new MockRM(conf);
rm.Start();
string nmVersion = "1.9.9";
ResourceTrackerService resourceTrackerService = rm.GetResourceTrackerService();
RegisterNodeManagerRequest req = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <RegisterNodeManagerRequest
>();
NodeId nodeId = NodeId.NewInstance("host2", 1234);
Resource capability = BuilderUtils.NewResource(1024, 1);
req.SetResource(capability);
req.SetNodeId(nodeId);
req.SetHttpPort(1234);
req.SetNMVersion(nmVersion);
// trying to register a invalid node.
RegisterNodeManagerResponse response = resourceTrackerService.RegisterNodeManager
(req);
NUnit.Framework.Assert.AreEqual(NodeAction.Shutdown, response.GetNodeAction());
NUnit.Framework.Assert.IsTrue("Diagnostic message did not contain: 'Disallowed NodeManager "
+ "Version " + nmVersion + ", is less than the minimum version'", response.GetDiagnosticsMessage
().Contains("Disallowed NodeManager Version " + nmVersion + ", is less than the minimum version "
));
}
public virtual void TestAdminRefreshQueuesWithFileSystemBasedConfigurationProvider
()
{
configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
);
//upload default configurations
UploadDefaultConfiguration();
try
{
rm = new MockRM(configuration);
rm.Init(configuration);
rm.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Should not get any exceptions");
}
CapacityScheduler cs = (CapacityScheduler)rm.GetRMContext().GetScheduler();
int maxAppsBefore = cs.GetConfiguration().GetMaximumSystemApplications();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
csConf.Set("yarn.scheduler.capacity.maximum-applications", "5000");
UploadConfiguration(csConf, "capacity-scheduler.xml");
rm.adminService.RefreshQueues(RefreshQueuesRequest.NewInstance());
int maxAppsAfter = cs.GetConfiguration().GetMaximumSystemApplications();
NUnit.Framework.Assert.AreEqual(maxAppsAfter, 5000);
NUnit.Framework.Assert.IsTrue(maxAppsAfter != maxAppsBefore);
}
public virtual void TestHandleContainerStatusInvalidCompletions()
{
rm = new MockRM(new YarnConfiguration());
rm.Start();
EventHandler handler = Org.Mockito.Mockito.Spy(rm.GetRMContext().GetDispatcher().
GetEventHandler());
// Case 1: Unmanaged AM
RMApp app = rm.SubmitApp(1024, true);
// Case 1.1: AppAttemptId is null
NMContainerStatus report = NMContainerStatus.NewInstance(ContainerId.NewContainerId
(ApplicationAttemptId.NewInstance(app.GetApplicationId(), 2), 1), ContainerState
.Complete, Resource.NewInstance(1024, 1), "Dummy Completed", 0, Priority.NewInstance
(10), 1234);
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
// Case 1.2: Master container is null
RMAppAttemptImpl currentAttempt = (RMAppAttemptImpl)app.GetCurrentAppAttempt();
currentAttempt.SetMasterContainer(null);
report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(currentAttempt.
GetAppAttemptId(), 0), ContainerState.Complete, Resource.NewInstance(1024, 1), "Dummy Completed"
, 0, Priority.NewInstance(10), 1234);
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
// Case 2: Managed AM
app = rm.SubmitApp(1024);
// Case 2.1: AppAttemptId is null
report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(ApplicationAttemptId
.NewInstance(app.GetApplicationId(), 2), 1), ContainerState.Complete, Resource.NewInstance
(1024, 1), "Dummy Completed", 0, Priority.NewInstance(10), 1234);
try
{
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
}
catch (Exception)
{
}
// expected - ignore
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
// Case 2.2: Master container is null
currentAttempt = (RMAppAttemptImpl)app.GetCurrentAppAttempt();
currentAttempt.SetMasterContainer(null);
report = NMContainerStatus.NewInstance(ContainerId.NewContainerId(currentAttempt.
GetAppAttemptId(), 0), ContainerState.Complete, Resource.NewInstance(1024, 1), "Dummy Completed"
, 0, Priority.NewInstance(10), 1234);
try
{
rm.GetResourceTrackerService().HandleNMContainerStatus(report, null);
}
catch (Exception)
{
}
// expected - ignore
Org.Mockito.Mockito.Verify(handler, Org.Mockito.Mockito.Never()).Handle((Org.Apache.Hadoop.Yarn.Event.Event
)Matchers.Any());
}
/// <exception cref="System.Exception"/>
public virtual void TestInvalidContainerReleaseRequest()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(1024);
// kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
am1.RegisterAppAttempt();
am1.AddRequests(new string[] { "127.0.0.1" }, Gb, 1, 1);
AllocateResponse alloc1Response = am1.Schedule();
// send the request
// kick the scheduler
nm1.NodeHeartbeat(true);
while (alloc1Response.GetAllocatedContainers().Count < 1)
{
Log.Info("Waiting for containers to be created for app 1...");
Sharpen.Thread.Sleep(1000);
alloc1Response = am1.Schedule();
}
NUnit.Framework.Assert.IsTrue(alloc1Response.GetAllocatedContainers().Count > 0);
RMApp app2 = rm.SubmitApp(1024);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt2 = app2.GetCurrentAppAttempt();
MockAM am2 = rm.SendAMLaunched(attempt2.GetAppAttemptId());
am2.RegisterAppAttempt();
// Now trying to release container allocated for app1 -> appAttempt1.
ContainerId cId = alloc1Response.GetAllocatedContainers()[0].GetId();
am2.AddContainerToBeReleased(cId);
try
{
am2.Schedule();
NUnit.Framework.Assert.Fail("Exception was expected!!");
}
catch (InvalidContainerReleaseException e)
{
StringBuilder sb = new StringBuilder("Cannot release container : ");
sb.Append(cId.ToString());
sb.Append(" not belonging to this application attempt : ");
sb.Append(attempt2.GetAppAttemptId().ToString());
NUnit.Framework.Assert.IsTrue(e.Message.Contains(sb.ToString()));
}
}
finally
{
if (rm != null)
{
rm.Stop();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestAppOnMultiNode()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
conf.Set("yarn.scheduler.capacity.node-locality-delay", "-1");
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
MockNM nm2 = rm.RegisterNode("h2:5678", 10240);
RMApp app = rm.SubmitApp(2000);
//kick the scheduling
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
//request for containers
int request = 13;
am.Allocate("h1", 1000, request, new AList <ContainerId>());
//kick the scheduler
IList <Container> conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId
>()).GetAllocatedContainers();
int contReceived = conts.Count;
while (contReceived < 3)
{
//only 3 containers are available on node1
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 3);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(3, conts.Count);
//send node2 heartbeat
conts = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
contReceived = conts.Count;
while (contReceived < 10)
{
nm2.NodeHeartbeat(true);
Sharpen.Collections.AddAll(conts, am.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers());
contReceived = conts.Count;
Log.Info("Got " + contReceived + " containers. Waiting to get " + 10);
Sharpen.Thread.Sleep(WaitSleepMs);
}
NUnit.Framework.Assert.AreEqual(10, conts.Count);
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
rm.Stop();
}
// Other stuff is verified in the regular web-services related tests
/// <summary>Test to verify the following RM HA transitions to the following states.</summary>
/// <remarks>
/// Test to verify the following RM HA transitions to the following states.
/// 1. Standby: Should be a no-op
/// 2. Active: Active services should start
/// 3. Active: Should be a no-op.
/// While active, submit a couple of jobs
/// 4. Standby: Active services should stop
/// 5. Active: Active services should start
/// 6. Stop the RM: All services should stop and RM should not be ready to
/// become Active
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestFailoverAndTransitions()
{
configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
Configuration conf = new YarnConfiguration(configuration);
rm = new MockRM(conf);
rm.Init(conf);
HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo
(HAServiceProtocol.RequestSource.RequestByUser);
NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing
, rm.adminService.GetServiceStatus().GetState());
NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started"
, rm.adminService.GetServiceStatus().IsReadyToBecomeActive());
CheckMonitorHealth();
rm.Start();
CheckMonitorHealth();
CheckStandbyRMFunctionality();
VerifyClusterMetrics(0, 0, 0, 0, 0, 0);
// 1. Transition to Standby - must be a no-op
rm.adminService.TransitionToStandby(requestInfo);
CheckMonitorHealth();
CheckStandbyRMFunctionality();
VerifyClusterMetrics(0, 0, 0, 0, 0, 0);
// 2. Transition to active
rm.adminService.TransitionToActive(requestInfo);
CheckMonitorHealth();
CheckActiveRMFunctionality();
VerifyClusterMetrics(1, 1, 1, 1, 2048, 1);
// 3. Transition to active - no-op
rm.adminService.TransitionToActive(requestInfo);
CheckMonitorHealth();
CheckActiveRMFunctionality();
VerifyClusterMetrics(1, 2, 2, 2, 2048, 2);
// 4. Transition to standby
rm.adminService.TransitionToStandby(requestInfo);
CheckMonitorHealth();
CheckStandbyRMFunctionality();
VerifyClusterMetrics(0, 0, 0, 0, 0, 0);
// 5. Transition to active to check Active->Standby->Active works
rm.adminService.TransitionToActive(requestInfo);
CheckMonitorHealth();
CheckActiveRMFunctionality();
VerifyClusterMetrics(1, 1, 1, 1, 2048, 1);
// 6. Stop the RM. All services should stop and RM should not be ready to
// become active
rm.Stop();
NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Stopping
, rm.adminService.GetServiceStatus().GetState());
NUnit.Framework.Assert.IsFalse("RM is ready to become active even after it is stopped"
, rm.adminService.GetServiceStatus().IsReadyToBecomeActive());
NUnit.Framework.Assert.IsFalse("Active RM services are started", rm.AreActiveServicesRunning
());
CheckMonitorHealth();
}
public virtual void TestRefreshUserToGroupsMappingsWithFileSystemBasedConfigurationProvider
()
{
configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
);
string[] defaultTestUserGroups = new string[] { "dummy_group1", "dummy_group2" };
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting("dummyUser",
defaultTestUserGroups);
string user = ugi.GetUserName();
IList <string> groupWithInit = new AList <string>(2);
for (int i = 0; i < ugi.GetGroupNames().Length; i++)
{
groupWithInit.AddItem(ugi.GetGroupNames()[i]);
}
// upload default configurations
UploadDefaultConfiguration();
Configuration conf = new Configuration();
conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(TestRMAdminService.MockUnixGroupsMapping
), typeof(GroupMappingServiceProvider));
UploadConfiguration(conf, "core-site.xml");
try
{
rm = new MockRM(configuration);
rm.Init(configuration);
rm.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Should not get any exceptions");
}
// Make sure RM will use the updated GroupMappingServiceProvider
IList <string> groupBefore = new AList <string>(Groups.GetUserToGroupsMappingService
(configuration).GetGroups(user));
NUnit.Framework.Assert.IsTrue(groupBefore.Contains("test_group_A") && groupBefore
.Contains("test_group_B") && groupBefore.Contains("test_group_C") && groupBefore
.Count == 3);
NUnit.Framework.Assert.IsTrue(groupWithInit.Count != groupBefore.Count);
NUnit.Framework.Assert.IsFalse(groupWithInit.Contains("test_group_A") || groupWithInit
.Contains("test_group_B") || groupWithInit.Contains("test_group_C"));
// update the groups
TestRMAdminService.MockUnixGroupsMapping.UpdateGroups();
rm.adminService.RefreshUserToGroupsMappings(RefreshUserToGroupsMappingsRequest.NewInstance
());
IList <string> groupAfter = Groups.GetUserToGroupsMappingService(configuration).GetGroups
(user);
// should get the updated groups
NUnit.Framework.Assert.IsTrue(groupAfter.Contains("test_group_D") && groupAfter.Contains
("test_group_E") && groupAfter.Contains("test_group_F") && groupAfter.Count == 3
);
}
public virtual void TestAuthorizedAccess()
{
TestAMAuthorization.MyContainerManager containerManager = new TestAMAuthorization.MyContainerManager
();
rm = new TestAMAuthorization.MockRMWithAMS(conf, containerManager);
rm.Start();
MockNM nm1 = rm.RegisterNode("localhost:1234", 5120);
IDictionary <ApplicationAccessType, string> acls = new Dictionary <ApplicationAccessType
, string>(2);
acls[ApplicationAccessType.ViewApp] = "*";
RMApp app = rm.SubmitApp(1024, "appname", "appuser", acls);
nm1.NodeHeartbeat(true);
int waitCount = 0;
while (containerManager.containerTokens == null && waitCount++ < 20)
{
Log.Info("Waiting for AM Launch to happen..");
Sharpen.Thread.Sleep(1000);
}
NUnit.Framework.Assert.IsNotNull(containerManager.containerTokens);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
ApplicationAttemptId applicationAttemptId = attempt.GetAppAttemptId();
WaitForLaunchedState(attempt);
// Create a client to the RM.
Configuration conf = rm.GetConfig();
YarnRPC rpc = YarnRPC.Create(conf);
UserGroupInformation currentUser = UserGroupInformation.CreateRemoteUser(applicationAttemptId
.ToString());
Credentials credentials = containerManager.GetContainerCredentials();
IPEndPoint rmBindAddress = rm.GetApplicationMasterService().GetBindAddress();
Org.Apache.Hadoop.Security.Token.Token <TokenIdentifier> amRMToken = TestAMAuthorization.MockRMWithAMS
.SetupAndReturnAMRMToken(rmBindAddress, credentials.GetAllTokens());
currentUser.AddToken(amRMToken);
ApplicationMasterProtocol client = currentUser.DoAs(new _PrivilegedAction_206(this
, rpc, conf));
RegisterApplicationMasterRequest request = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<RegisterApplicationMasterRequest>();
RegisterApplicationMasterResponse response = client.RegisterApplicationMaster(request
);
NUnit.Framework.Assert.IsNotNull(response.GetClientToAMTokenMasterKey());
if (UserGroupInformation.IsSecurityEnabled())
{
NUnit.Framework.Assert.IsTrue(((byte[])response.GetClientToAMTokenMasterKey().Array
()).Length > 0);
}
NUnit.Framework.Assert.AreEqual("Register response has bad ACLs", "*", response.GetApplicationACLs
()[ApplicationAccessType.ViewApp]);
}
public virtual void TestDecommissionWithExcludeHosts()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNodesExcludeFilePath, hostFile.GetAbsolutePath());
WriteToHostsFile(string.Empty);
DrainDispatcher dispatcher = new DrainDispatcher();
rm = new _MockRM_162(dispatcher, conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
MockNM nm2 = rm.RegisterNode("host2:5678", 10240);
MockNM nm3 = rm.RegisterNode("localhost:4433", 1024);
dispatcher.Await();
int metricCount = ClusterMetrics.GetMetrics().GetNumDecommisionedNMs();
NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
dispatcher.Await();
// To test that IPs also work
string ip = NetUtils.NormalizeHostName("localhost");
WriteToHostsFile("host2", ip);
rm.GetNodesListManager().RefreshNodes(conf);
CheckDecommissionedNMCount(rm, metricCount + 2);
nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue("The decommisioned metrics are not updated", NodeAction
.Shutdown.Equals(nodeHeartbeat.GetNodeAction()));
nodeHeartbeat = nm3.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue("The decommisioned metrics are not updated", NodeAction
.Shutdown.Equals(nodeHeartbeat.GetNodeAction()));
dispatcher.Await();
WriteToHostsFile(string.Empty);
rm.GetNodesListManager().RefreshNodes(conf);
nm3 = rm.RegisterNode("localhost:4433", 1024);
dispatcher.Await();
nodeHeartbeat = nm3.NodeHeartbeat(true);
dispatcher.Await();
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
// decommissined node is 1 since 1 node is rejoined after updating exclude
// file
CheckDecommissionedNMCount(rm, metricCount + 1);
}
// This is to test AM Host and rpc port are invalidated after the am attempt
// is killed or failed, so that client doesn't get the wrong information.
/// <exception cref="System.Exception"/>
public virtual void TestInvalidateAMHostPortWhenAMFailedOrKilled()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MockRM rm1 = new MockRM(conf);
rm1.Start();
// a succeeded app
RMApp app1 = rm1.SubmitApp(200);
MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm1, nm1, am1);
// a failed app
RMApp app2 = rm1.SubmitApp(200);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(am2.GetApplicationAttemptId(), 1, ContainerState.Complete);
am2.WaitForState(RMAppAttemptState.Failed);
rm1.WaitForState(app2.GetApplicationId(), RMAppState.Failed);
// a killed app
RMApp app3 = rm1.SubmitApp(200);
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm1);
rm1.KillApp(app3.GetApplicationId());
rm1.WaitForState(app3.GetApplicationId(), RMAppState.Killed);
rm1.WaitForState(am3.GetApplicationAttemptId(), RMAppAttemptState.Killed);
GetApplicationsRequest request1 = GetApplicationsRequest.NewInstance(EnumSet.Of(YarnApplicationState
.Finished, YarnApplicationState.Killed, YarnApplicationState.Failed));
GetApplicationsResponse response1 = rm1.GetClientRMService().GetApplications(request1
);
IList <ApplicationReport> appList1 = response1.GetApplicationList();
NUnit.Framework.Assert.AreEqual(3, appList1.Count);
foreach (ApplicationReport report in appList1)
{
// killed/failed apps host and rpc port are invalidated.
if (report.GetApplicationId().Equals(app2.GetApplicationId()) || report.GetApplicationId
().Equals(app3.GetApplicationId()))
{
NUnit.Framework.Assert.AreEqual("N/A", report.GetHost());
NUnit.Framework.Assert.AreEqual(-1, report.GetRpcPort());
}
// succeeded app's host and rpc port is not invalidated
if (report.GetApplicationId().Equals(app1.GetApplicationId()))
{
NUnit.Framework.Assert.IsFalse(report.GetHost().Equals("N/A"));
NUnit.Framework.Assert.IsTrue(report.GetRpcPort() != -1);
}
}
}
// Test even if AM container is allocated with containerId not equal to 1, the
// following allocate requests from AM should be able to retrieve the
// corresponding NM Token.
/// <exception cref="System.Exception"/>
public virtual void TestNMTokenSentForNormalContainer()
{
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName
());
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
// Call getNewContainerId to increase container Id so that the AM container
// Id doesn't equal to one.
CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId();
// kick the scheduling
nm1.NodeHeartbeat(true);
MockAM am = MockRM.LaunchAM(app, rm, nm1);
// am container Id not equal to 1.
NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId
() != 1);
// NMSecretManager doesn't record the node on which the am is allocated.
NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
int NumContainers = 1;
IList <Container> containers = new AList <Container>();
// nmTokens keeps track of all the nmTokens issued in the allocate call.
IList <NMToken> expectedNMTokens = new AList <NMToken>();
// am1 allocate 1 container on nm1.
while (true)
{
AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList
<ContainerId>());
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
if (containers.Count == NumContainers)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
NodeId nodeId = expectedNMTokens[0].GetNodeId();
// NMToken is sent for the allocated container.
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId);
}
public virtual void TestSetRMIdentifierInRegistration()
{
Configuration conf = new Configuration();
rm = new MockRM(conf);
rm.Start();
MockNM nm = new MockNM("host1:1234", 5120, rm.GetResourceTrackerService());
RegisterNodeManagerResponse response = nm.RegisterNode();
// Verify the RMIdentifier is correctly set in RegisterNodeManagerResponse
NUnit.Framework.Assert.AreEqual(ResourceManager.GetClusterTimeStamp(), response.GetRMIdentifier
());
}
/// <exception cref="System.Exception"/>
public virtual void TestAppCleanupWhenRMRestartedBeforeAppFinished()
{
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// start RM
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 1024, rm1.GetResourceTrackerService());
nm1.RegisterNode();
MockNM nm2 = new MockNM("127.0.0.1:5678", 1024, rm1.GetResourceTrackerService());
nm2.RegisterNode();
// create app and launch the AM
RMApp app0 = rm1.SubmitApp(200);
MockAM am0 = LaunchAM(app0, rm1, nm1);
// alloc another container on nm2
AllocateResponse allocResponse = am0.Allocate(Arrays.AsList(ResourceRequest.NewInstance
(Priority.NewInstance(1), "*", Resource.NewInstance(1024, 0), 1)), null);
while (null == allocResponse.GetAllocatedContainers() || allocResponse.GetAllocatedContainers
().IsEmpty())
{
nm2.NodeHeartbeat(true);
allocResponse = am0.Allocate(null, null);
Sharpen.Thread.Sleep(1000);
}
// start new RM
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
// nm1/nm2 register to rm2, and do a heartbeat
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode(Arrays.AsList(NMContainerStatus.NewInstance(ContainerId.NewContainerId
(am0.GetApplicationAttemptId(), 1), ContainerState.Complete, Resource.NewInstance
(1024, 1), string.Empty, 0, Priority.NewInstance(0), 1234)), Arrays.AsList(app0.
GetApplicationId()));
nm2.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm2.RegisterNode(Arrays.AsList(app0.GetApplicationId()));
// assert app state has been saved.
rm2.WaitForState(app0.GetApplicationId(), RMAppState.Failed);
// wait for application cleanup message received on NM1
WaitForAppCleanupMessageRecved(nm1, app0.GetApplicationId());
// wait for application cleanup message received on NM2
WaitForAppCleanupMessageRecved(nm2, app0.GetApplicationId());
rm1.Stop();
rm2.Stop();
}
public virtual void TestRefreshSuperUserGroupsWithFileSystemBasedConfigurationProvider
()
{
configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
);
//upload default configurations
UploadDefaultConfiguration();
try
{
rm = new MockRM(configuration);
rm.Init(configuration);
rm.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Should not get any exceptions");
}
Configuration coreConf = new Configuration(false);
coreConf.Set("hadoop.proxyuser.test.groups", "test_groups");
coreConf.Set("hadoop.proxyuser.test.hosts", "test_hosts");
UploadConfiguration(coreConf, "core-site.xml");
rm.adminService.RefreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequest
.NewInstance());
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
()["hadoop.proxyuser.test.groups"].Count == 1);
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
()["hadoop.proxyuser.test.groups"].Contains("test_groups"));
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
()["hadoop.proxyuser.test.hosts"].Count == 1);
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
()["hadoop.proxyuser.test.hosts"].Contains("test_hosts"));
Configuration yarnConf = new Configuration(false);
yarnConf.Set("yarn.resourcemanager.proxyuser.test.groups", "test_groups_1");
yarnConf.Set("yarn.resourcemanager.proxyuser.test.hosts", "test_hosts_1");
UploadConfiguration(yarnConf, "yarn-site.xml");
// RM specific configs will overwrite the common ones
rm.adminService.RefreshSuperUserGroupsConfiguration(RefreshSuperUserGroupsConfigurationRequest
.NewInstance());
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
()["hadoop.proxyuser.test.groups"].Count == 1);
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyGroups
()["hadoop.proxyuser.test.groups"].Contains("test_groups_1"));
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
()["hadoop.proxyuser.test.hosts"].Count == 1);
NUnit.Framework.Assert.IsTrue(ProxyUsers.GetDefaultImpersonationProvider().GetProxyHosts
()["hadoop.proxyuser.test.hosts"].Contains("test_hosts_1"));
}
public virtual void TestRefreshNodesWithLocalConfigurationProvider()
{
rm = new MockRM(configuration);
rm.Init(configuration);
rm.Start();
try
{
rm.adminService.RefreshNodes(RefreshNodesRequest.NewInstance());
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Using localConfigurationProvider. Should not get any exception."
);
}
}
public virtual void TestGetNewAppId()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
MockRM rm = new MockRM(conf);
rm.Start();
GetNewApplicationResponse resp = rm.GetNewAppId();
System.Diagnostics.Debug.Assert((resp.GetApplicationId().GetId() != 0));
System.Diagnostics.Debug.Assert((resp.GetMaximumResourceCapability().GetMemory()
> 0));
rm.Stop();
}
/// <exception cref="System.IO.IOException"/>
protected internal virtual void StartRMs(MockRM rm1, Configuration confForRM1, MockRM
rm2, Configuration confForRM2)
{
rm1.Init(confForRM1);
rm1.Start();
NUnit.Framework.Assert.IsTrue(rm1.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState
.Standby);
rm2.Init(confForRM2);
rm2.Start();
NUnit.Framework.Assert.IsTrue(rm2.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState
.Standby);
rm1.adminService.TransitionToActive(requestInfo);
NUnit.Framework.Assert.IsTrue(rm1.GetRMContext().GetHAServiceState() == HAServiceProtocol.HAServiceState
.Active);
}
public virtual void TestRMDispatcherForHA()
{
string errorMessageForEventHandler = "Expect to get the same number of handlers";
string errorMessageForService = "Expect to get the same number of services";
configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
Configuration conf = new YarnConfiguration(configuration);
rm = new _MockRM_313(conf);
rm.Init(conf);
int expectedEventHandlerCount = ((TestRMHA.MyCountingDispatcher)rm.GetRMContext()
.GetDispatcher()).GetEventHandlerCount();
int expectedServiceCount = rm.GetServices().Count;
NUnit.Framework.Assert.IsTrue(expectedEventHandlerCount != 0);
HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo
(HAServiceProtocol.RequestSource.RequestByUser);
NUnit.Framework.Assert.AreEqual(StateErr, HAServiceProtocol.HAServiceState.Initializing
, rm.adminService.GetServiceStatus().GetState());
NUnit.Framework.Assert.IsFalse("RM is ready to become active before being started"
, rm.adminService.GetServiceStatus().IsReadyToBecomeActive());
rm.Start();
//call transitions to standby and active a couple of times
rm.adminService.TransitionToStandby(requestInfo);
rm.adminService.TransitionToActive(requestInfo);
rm.adminService.TransitionToStandby(requestInfo);
rm.adminService.TransitionToActive(requestInfo);
rm.adminService.TransitionToStandby(requestInfo);
TestRMHA.MyCountingDispatcher dispatcher = (TestRMHA.MyCountingDispatcher)rm.GetRMContext
().GetDispatcher();
NUnit.Framework.Assert.IsTrue(!dispatcher.IsStopped());
rm.adminService.TransitionToActive(requestInfo);
NUnit.Framework.Assert.AreEqual(errorMessageForEventHandler, expectedEventHandlerCount
, ((TestRMHA.MyCountingDispatcher)rm.GetRMContext().GetDispatcher()).GetEventHandlerCount
());
NUnit.Framework.Assert.AreEqual(errorMessageForService, expectedServiceCount, rm.
GetServices().Count);
// Keep the dispatcher reference before transitioning to standby
dispatcher = (TestRMHA.MyCountingDispatcher)rm.GetRMContext().GetDispatcher();
rm.adminService.TransitionToStandby(requestInfo);
NUnit.Framework.Assert.AreEqual(errorMessageForEventHandler, expectedEventHandlerCount
, ((TestRMHA.MyCountingDispatcher)rm.GetRMContext().GetDispatcher()).GetEventHandlerCount
());
NUnit.Framework.Assert.AreEqual(errorMessageForService, expectedServiceCount, rm.
GetServices().Count);
NUnit.Framework.Assert.IsTrue(dispatcher.IsStopped());
rm.Stop();
}
/// <summary>
/// Test RM read NM next heartBeat Interval correctly from Configuration file,
/// and NM get next heartBeat Interval from RM correctly
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestGetNextHeartBeatInterval()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNmHeartbeatIntervalMs, "4000");
rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
MockNM nm2 = rm.RegisterNode("host2:5678", 10240);
NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual(4000, nodeHeartbeat.GetNextHeartBeatInterval());
NodeHeartbeatResponse nodeHeartbeat2 = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.AreEqual(4000, nodeHeartbeat2.GetNextHeartBeatInterval());
}
/// <exception cref="System.Exception"/>
public virtual void TestMoveSuccessful()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
RMApp app = rm1.SubmitApp(1024);
ClientRMService clientRMService = rm1.GetClientRMService();
// FIFO scheduler does not support moves
clientRMService.MoveApplicationAcrossQueues(MoveApplicationAcrossQueuesRequest.NewInstance
(app.GetApplicationId(), "newqueue"));
RMApp rmApp = rm1.GetRMContext().GetRMApps()[app.GetApplicationId()];
NUnit.Framework.Assert.AreEqual("newqueue", rmApp.GetQueue());
rm1.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestUsageWithOneAttemptAndOneContainer()
{
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm = new MockNM("127.0.0.1:1234", 15120, rm.GetResourceTrackerService());
nm.RegisterNode();
RMApp app0 = rm.SubmitApp(200);
RMAppMetrics rmAppMetrics = app0.GetRMAppMetrics();
NUnit.Framework.Assert.IsTrue("Before app submittion, memory seconds should have been 0 but was "
+ rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() == 0);
NUnit.Framework.Assert.IsTrue("Before app submission, vcore seconds should have been 0 but was "
+ rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() == 0);
RMAppAttempt attempt0 = app0.GetCurrentAppAttempt();
nm.NodeHeartbeat(true);
MockAM am0 = rm.SendAMLaunched(attempt0.GetAppAttemptId());
am0.RegisterAppAttempt();
RMContainer rmContainer = rm.GetResourceScheduler().GetRMContainer(attempt0.GetMasterContainer
().GetId());
// Allow metrics to accumulate.
int sleepInterval = 1000;
int cumulativeSleepTime = 0;
while (rmAppMetrics.GetMemorySeconds() <= 0 && cumulativeSleepTime < 5000)
{
Sharpen.Thread.Sleep(sleepInterval);
cumulativeSleepTime += sleepInterval;
}
rmAppMetrics = app0.GetRMAppMetrics();
NUnit.Framework.Assert.IsTrue("While app is running, memory seconds should be >0 but is "
+ rmAppMetrics.GetMemorySeconds(), rmAppMetrics.GetMemorySeconds() > 0);
NUnit.Framework.Assert.IsTrue("While app is running, vcore seconds should be >0 but is "
+ rmAppMetrics.GetVcoreSeconds(), rmAppMetrics.GetVcoreSeconds() > 0);
MockRM.FinishAMAndVerifyAppState(app0, rm, nm, am0);
AggregateAppResourceUsage ru = CalculateContainerResourceMetrics(rmContainer);
rmAppMetrics = app0.GetRMAppMetrics();
NUnit.Framework.Assert.AreEqual("Unexcpected MemorySeconds value", ru.GetMemorySeconds
(), rmAppMetrics.GetMemorySeconds());
NUnit.Framework.Assert.AreEqual("Unexpected VcoreSeconds value", ru.GetVcoreSeconds
(), rmAppMetrics.GetVcoreSeconds());
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestResourceTypes()
{
Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes> >
driver = new Dictionary <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
> >();
CapacitySchedulerConfiguration csconf = new CapacitySchedulerConfiguration();
csconf.SetResourceComparator(typeof(DominantResourceCalculator));
YarnConfiguration testCapacityDRConf = new YarnConfiguration(csconf);
testCapacityDRConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
), typeof(ResourceScheduler));
YarnConfiguration testCapacityDefConf = new YarnConfiguration();
testCapacityDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler
), typeof(ResourceScheduler));
YarnConfiguration testFairDefConf = new YarnConfiguration();
testFairDefConf.SetClass(YarnConfiguration.RmScheduler, typeof(FairScheduler), typeof(
ResourceScheduler));
driver[conf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory);
driver[testCapacityDRConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.
Cpu, YarnServiceProtos.SchedulerResourceTypes.Memory);
driver[testCapacityDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes
.Memory);
driver[testFairDefConf] = EnumSet.Of(YarnServiceProtos.SchedulerResourceTypes.Memory
, YarnServiceProtos.SchedulerResourceTypes.Cpu);
foreach (KeyValuePair <YarnConfiguration, EnumSet <YarnServiceProtos.SchedulerResourceTypes
> > entry in driver)
{
EnumSet <YarnServiceProtos.SchedulerResourceTypes> expectedValue = entry.Value;
MockRM rm = new MockRM(entry.Key);
rm.Start();
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
RMApp app1 = rm.SubmitApp(2048);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = rm.SendAMLaunched(attempt1.GetAppAttemptId());
RegisterApplicationMasterResponse resp = am1.RegisterAppAttempt();
EnumSet <YarnServiceProtos.SchedulerResourceTypes> types = resp.GetSchedulerResourceTypes
();
Log.Info("types = " + types.ToString());
NUnit.Framework.Assert.AreEqual(expectedValue, types);
rm.Stop();
}
}
public virtual void TestFailoverClearsRMContext()
{
configuration.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
configuration.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
Configuration conf = new YarnConfiguration(configuration);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
// 1. start RM
rm = new MockRM(conf, memStore);
rm.Init(conf);
rm.Start();
HAServiceProtocol.StateChangeRequestInfo requestInfo = new HAServiceProtocol.StateChangeRequestInfo
(HAServiceProtocol.RequestSource.RequestByUser);
CheckMonitorHealth();
CheckStandbyRMFunctionality();
// 2. Transition to active
rm.adminService.TransitionToActive(requestInfo);
CheckMonitorHealth();
CheckActiveRMFunctionality();
VerifyClusterMetrics(1, 1, 1, 1, 2048, 1);
NUnit.Framework.Assert.AreEqual(1, rm.GetRMContext().GetRMNodes().Count);
NUnit.Framework.Assert.AreEqual(1, rm.GetRMContext().GetRMApps().Count);
// 3. Create new RM
rm = new _MockRM_550(conf, memStore);
rm.Init(conf);
rm.Start();
CheckMonitorHealth();
CheckStandbyRMFunctionality();
// 4. Try Transition to active, throw exception
try
{
rm.adminService.TransitionToActive(requestInfo);
NUnit.Framework.Assert.Fail("Transitioned to Active should throw exception.");
}
catch (Exception e)
{
NUnit.Framework.Assert.IsTrue("Error when transitioning to Active mode".Contains(
e.Message));
}
// 5. Clears the metrics
VerifyClusterMetrics(0, 0, 0, 0, 0, 0);
NUnit.Framework.Assert.AreEqual(0, rm.GetRMContext().GetRMNodes().Count);
NUnit.Framework.Assert.AreEqual(0, rm.GetRMContext().GetRMApps().Count);
}
public virtual void TestDecommissionWithIncludeHosts()
{
WriteToHostsFile("localhost", "host1", "host2");
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.RmNodesIncludeFilePath, hostFile.GetAbsolutePath());
rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("host1:1234", 5120);
MockNM nm2 = rm.RegisterNode("host2:5678", 10240);
MockNM nm3 = rm.RegisterNode("localhost:4433", 1024);
ClusterMetrics metrics = ClusterMetrics.GetMetrics();
System.Diagnostics.Debug.Assert((metrics != null));
int metricCount = metrics.GetNumDecommisionedNMs();
NodeHeartbeatResponse nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
nodeHeartbeat = nm3.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
// To test that IPs also work
string ip = NetUtils.NormalizeHostName("localhost");
WriteToHostsFile("host1", ip);
rm.GetNodesListManager().RefreshNodes(conf);
CheckDecommissionedNMCount(rm, ++metricCount);
nodeHeartbeat = nm1.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
NUnit.Framework.Assert.AreEqual(1, ClusterMetrics.GetMetrics().GetNumDecommisionedNMs
());
nodeHeartbeat = nm2.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue("Node is not decommisioned.", NodeAction.Shutdown.Equals
(nodeHeartbeat.GetNodeAction()));
nodeHeartbeat = nm3.NodeHeartbeat(true);
NUnit.Framework.Assert.IsTrue(NodeAction.Normal.Equals(nodeHeartbeat.GetNodeAction
()));
NUnit.Framework.Assert.AreEqual(metricCount, ClusterMetrics.GetMetrics().GetNumDecommisionedNMs
());
}
/// <exception cref="System.Exception"/>
public virtual void TestFinishApplicationMasterBeforeRegistering()
{
MockRM rm = new MockRM(conf);
try
{
rm.Start();
// Register node1
MockNM nm1 = rm.RegisterNode("127.0.0.1:1234", 6 * Gb);
// Submit an application
RMApp app1 = rm.SubmitApp(2048);
MockAM am1 = MockRM.LaunchAM(app1, rm, nm1);
FinishApplicationMasterRequest req = FinishApplicationMasterRequest.NewInstance(FinalApplicationStatus
.Failed, string.Empty, string.Empty);
try
{
am1.UnregisterAppAttempt(req, false);
NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
);
}
catch (ApplicationMasterNotRegisteredException e)
{
NUnit.Framework.Assert.IsNotNull(e);
NUnit.Framework.Assert.IsNotNull(e.Message);
NUnit.Framework.Assert.IsTrue(e.Message.Contains("Application Master is trying to unregister before registering for:"
));
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("ApplicationMasterNotRegisteredException should be thrown"
);
}
am1.RegisterAppAttempt();
am1.UnregisterAppAttempt(req, false);
am1.WaitForState(RMAppAttemptState.Finishing);
}
finally
{
if (rm != null)
{
rm.Stop();
}
}
}