public virtual void TestLogAggregationContextPassedIntoContainerToken()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("127.0.0.1:1234", 8000);
MockNM nm2 = rm1.RegisterNode("127.0.0.1:2345", 8000);
// LogAggregationContext is set as null
NUnit.Framework.Assert.IsNull(GetLogAggregationContextFromContainerToken(rm1, nm1
, null));
// create a not-null LogAggregationContext
LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern"
, "excludePattern", "rolledLogsIncludePattern", "rolledLogsExcludePattern");
LogAggregationContext returned = GetLogAggregationContextFromContainerToken(rm1,
nm2, logAggregationContext);
NUnit.Framework.Assert.AreEqual("includePattern", returned.GetIncludePattern());
NUnit.Framework.Assert.AreEqual("excludePattern", returned.GetExcludePattern());
NUnit.Framework.Assert.AreEqual("rolledLogsIncludePattern", returned.GetRolledLogsIncludePattern
());
NUnit.Framework.Assert.AreEqual("rolledLogsExcludePattern", returned.GetRolledLogsExcludePattern
());
rm1.Stop();
}
/// <exception cref="System.IO.IOException"/>
private void CreateClusterWithQueuesAndOneNode(int mem, int vCores, string policy
)
{
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine("<queue name=\"root\" >");
@out.WriteLine(" <queue name=\"parentA\" >");
@out.WriteLine(" <weight>8</weight>");
@out.WriteLine(" <queue name=\"childA1\" />");
@out.WriteLine(" <queue name=\"childA2\" />");
@out.WriteLine(" <queue name=\"childA3\" />");
@out.WriteLine(" <queue name=\"childA4\" />");
@out.WriteLine(" </queue>");
@out.WriteLine(" <queue name=\"parentB\" >");
@out.WriteLine(" <weight>1</weight>");
@out.WriteLine(" <queue name=\"childB1\" />");
@out.WriteLine(" <queue name=\"childB2\" />");
@out.WriteLine(" </queue>");
@out.WriteLine("</queue>");
@out.WriteLine("<defaultQueueSchedulingPolicy>" + policy + "</defaultQueueSchedulingPolicy>"
);
@out.WriteLine("</allocations>");
@out.Close();
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(mem, vCores), 1,
"127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
}
public virtual void TestNormalContainerAllocationWhenDNSUnavailable()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
RMApp app1 = rm1.SubmitApp(200);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
// acquire the container.
SecurityUtilTestHelper.SetTokenServiceUseIp(true);
IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
// not able to fetch the container;
NUnit.Framework.Assert.AreEqual(0, containers.Count);
SecurityUtilTestHelper.SetTokenServiceUseIp(false);
containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>())
.GetAllocatedContainers();
// should be able to fetch the container;
NUnit.Framework.Assert.AreEqual(1, containers.Count);
}
public virtual void SetUp()
{
dispatcher = new DrainDispatcher();
this.rm = new _MockRM_58(this);
rm.Start();
amService = rm.GetApplicationMasterService();
}
// Test regular RM restart/failover, new RM should not count
// AM failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestRMRestartOrFailoverNotCountedForAMFailures()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
// AM should be restarted even though max-am-attempt is 1.
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt1).MayBeLastAttempt());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
rm2.Start();
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
// re-register the NM
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus
>();
status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager);
status.SetContainerId(attempt1.GetMasterContainer().GetId());
status.SetContainerState(ContainerState.Complete);
status.SetDiagnostics(string.Empty);
nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null);
rm2.WaitForState(attempt1.GetAppAttemptId(), RMAppAttemptState.Failed);
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledByResourcemanager, appState
.GetAttempt(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Will automatically start a new AppAttempt in rm2
rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt3 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt3.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
public virtual void TestContainerTokenGeneratedOnPullRequest()
{
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("127.0.0.1:1234", 8000);
RMApp app1 = rm1.SubmitApp(200);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
RMContainer container = rm1.GetResourceScheduler().GetRMContainer(containerId2);
// no container token is generated.
NUnit.Framework.Assert.AreEqual(containerId2, container.GetContainerId());
NUnit.Framework.Assert.IsNull(container.GetContainer().GetContainerToken());
// acquire the container.
IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
NUnit.Framework.Assert.AreEqual(containerId2, containers[0].GetId());
// container token is generated.
NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken());
rm1.Stop();
}
public virtual void TestUpdateMaxAllocationUsesTotal()
{
int configuredMaxVCores = 20;
int configuredMaxMemory = 10 * 1024;
Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(configuredMaxMemory, configuredMaxVCores);
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
);
Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(0, 0);
Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 5);
Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 10);
SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo"
, 8080));
Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource
);
Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1);
SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar"
, 8081));
Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource
);
Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
scheduler.nodes = new Dictionary <NodeId, SchedulerNode>();
scheduler.nodes[mockNode1.GetNodeID()] = mockNode1;
scheduler.UpdateMaximumAllocation(mockNode1, true);
VerifyMaximumResourceCapability(fullResource1, scheduler);
scheduler.nodes[mockNode2.GetNodeID()] = mockNode2;
scheduler.UpdateMaximumAllocation(mockNode2, true);
VerifyMaximumResourceCapability(fullResource2, scheduler);
Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID());
scheduler.UpdateMaximumAllocation(mockNode2, false);
VerifyMaximumResourceCapability(fullResource1, scheduler);
Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID());
scheduler.UpdateMaximumAllocation(mockNode1, false);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
}
finally
{
rm.Stop();
}
}
public virtual void TestExistenceOfResourceRequestInRMContainer()
{
Configuration conf = new Configuration();
MockRM rm1 = new MockRM(conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
RMApp app1 = rm1.SubmitApp(1024);
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
ResourceScheduler scheduler = rm1.GetResourceScheduler();
// request a container.
am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>());
ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId
(), 2);
rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated);
// Verify whether list of ResourceRequest is present in RMContainer
// while moving to ALLOCATED state
NUnit.Framework.Assert.IsNotNull(scheduler.GetRMContainer(containerId2).GetResourceRequests
());
// Allocate container
am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
rm1.WaitForState(nm1, containerId2, RMContainerState.Acquired);
// After RMContainer moving to ACQUIRED state, list of ResourceRequest will
// be empty
NUnit.Framework.Assert.IsNull(scheduler.GetRMContainer(containerId2).GetResourceRequests
());
}
/// <exception cref="System.Exception"/>
private void TestRMWritingMassiveHistory(bool isFS)
{
// 1. Show RM can run with writing history data
// 2. Test additional workload of processing history events
YarnConfiguration conf = new YarnConfiguration();
if (isFS)
{
conf.SetBoolean(FairSchedulerConfiguration.AssignMultiple, true);
conf.Set("yarn.resourcemanager.scheduler.class", typeof(FairScheduler).FullName);
}
else
{
conf.Set("yarn.resourcemanager.scheduler.class", typeof(CapacityScheduler).FullName
);
}
// don't process history events
MockRM rm = new _MockRM_399(conf);
long startTime1 = Runtime.CurrentTimeMillis();
TestRMWritingMassiveHistory(rm);
long finishTime1 = Runtime.CurrentTimeMillis();
long elapsedTime1 = finishTime1 - startTime1;
rm = new MockRM(conf);
long startTime2 = Runtime.CurrentTimeMillis();
TestRMWritingMassiveHistory(rm);
long finishTime2 = Runtime.CurrentTimeMillis();
long elapsedTime2 = finishTime2 - startTime2;
// No more than 10% additional workload
// Should be much less, but computation time is fluctuated
NUnit.Framework.Assert.IsTrue(elapsedTime2 - elapsedTime1 < elapsedTime1 / 10);
}
// This is to test fetching AM container will be retried, if AM container is
// not fetchable since DNS is unavailable causing container token/NMtoken
// creation failure.
/// <exception cref="System.Exception"/>
public virtual void TestAMContainerAllocationWhenDNSUnavailable()
{
MockRM rm1 = new _MockRM_303(this, conf);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000);
SecurityUtilTestHelper.SetTokenServiceUseIp(true);
RMApp app1 = rm1.SubmitApp(200);
RMAppAttempt attempt = app1.GetCurrentAppAttempt();
nm1.NodeHeartbeat(true);
// fetching am container will fail, keep retrying 5 times.
while (numRetries <= 5)
{
nm1.NodeHeartbeat(true);
Sharpen.Thread.Sleep(1000);
NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Scheduled, attempt.GetAppAttemptState
());
System.Console.Out.WriteLine("Waiting for am container to be allocated.");
}
SecurityUtilTestHelper.SetTokenServiceUseIp(false);
rm1.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Allocated);
MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
}
/// <exception cref="System.Exception"/>
private LogAggregationContext GetLogAggregationContextFromContainerToken(MockRM rm1
, MockNM nm1, LogAggregationContext logAggregationContext)
{
RMApp app2 = rm1.SubmitApp(200, logAggregationContext);
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1);
nm1.NodeHeartbeat(true);
// request a container.
am2.Allocate("127.0.0.1", 512, 1, new AList <ContainerId>());
ContainerId containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(
), 2);
rm1.WaitForState(nm1, containerId, RMContainerState.Allocated);
// acquire the container.
IList <Container> containers = am2.Allocate(new AList <ResourceRequest>(), new AList
<ContainerId>()).GetAllocatedContainers();
NUnit.Framework.Assert.AreEqual(containerId, containers[0].GetId());
// container token is generated.
NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken());
ContainerTokenIdentifier token = BuilderUtils.NewContainerTokenIdentifier(containers
[0].GetContainerToken());
return(token.GetLogAggregationContext());
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocateWithLabels()
{
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm1 = new _MockRM_638(this, GetConfigurationWithQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
// label = x
MockNM nm2 = rm1.RegisterNode("h2:1234", 8000);
// label = y
MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
// label = <empty>
ContainerId containerId;
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm3);
// request a container.
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h1");
// launch an app to queue b1 (label = y), and check all container will
// be allocated in h2
RMApp app2 = rm1.SubmitApp(200, "app", "user", null, "b1");
MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm3);
// request a container.
am2.Allocate("*", 1024, 1, new AList <ContainerId>(), "y");
containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am2.GetApplicationAttemptId(), containerId, rm1, "h2");
// launch an app to queue c1 (label = ""), and check all container will
// be allocated in h3
RMApp app3 = rm1.SubmitApp(200, "app", "user", null, "c1");
MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm3);
// request a container.
am3.Allocate("*", 1024, 1, new AList <ContainerId>());
containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState
.Allocated, 10 * 1000));
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3");
rm1.Close();
}
// Test RM restarts after AM container is preempted, new RM should not count
// AM preemption failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestPreemptedAMRestartOnRMRestart()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler();
ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 1);
// Forcibly preempt the am container;
scheduler.KillContainer(scheduler.GetRMContainer(amContainer));
am1.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry());
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// state store has 1 attempt stored.
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
NUnit.Framework.Assert.AreEqual(1, appState.GetAttemptCount());
// attempt stored has the preempted container exit status.
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Preempted, appState.GetAttempt
(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode();
rm2.Start();
// Restarted RM should re-launch the am.
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt2 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt2.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
public virtual void Setup()
{
Configuration conf = CreateConfiguration();
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
}
private void CheckUsedResource(MockRM rm, string queueName, int memory, string label
)
{
CapacityScheduler scheduler = (CapacityScheduler)rm.GetResourceScheduler();
CSQueue queue = scheduler.GetQueue(queueName);
NUnit.Framework.Assert.AreEqual(memory, queue.GetQueueResourceUsage().GetUsed(label
).GetMemory());
}
// To avoid using cached client
/// <exception cref="System.Exception"/>
public virtual void TestAMRMMasterKeysUpdate()
{
AtomicReference <AMRMTokenSecretManager> spySecretMgrRef = new AtomicReference <AMRMTokenSecretManager
>();
MockRM rm = new _MockRM_349(this, spySecretMgrRef, conf);
// Skip the login.
rm.Start();
MockNM nm = rm.RegisterNode("127.0.0.1:1234", 8000);
RMApp app = rm.SubmitApp(200);
MockAM am = MockRM.LaunchAndRegisterAM(app, rm, nm);
AMRMTokenSecretManager spySecretMgr = spySecretMgrRef.Get();
// Do allocate. Should not update AMRMToken
AllocateResponse response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<AllocateRequest>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> oldToken = rm.GetRMContext
().GetRMApps()[app.GetApplicationId()].GetRMAppAttempt(am.GetApplicationAttemptId
()).GetAMRMToken();
// roll over the master key
// Do allocate again. the AM should get the latest AMRMToken
rm.GetRMContext().GetAMRMTokenSecretManager().RollMasterKey();
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken());
Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = ConverterUtils
.ConvertFromYarn(response.GetAMRMToken(), new Text(response.GetAMRMToken().GetService
()));
NUnit.Framework.Assert.AreEqual(amrmToken.DecodeIdentifier().GetKeyId(), rm.GetRMContext
().GetAMRMTokenSecretManager().GetMasterKey().GetMasterKey().GetKeyId());
// Do allocate again with the same old token and verify the RM sends
// back the last generated token instead of generating it again.
Org.Mockito.Mockito.Reset(spySecretMgr);
UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(am.GetApplicationAttemptId
().ToString(), new string[0]);
ugi.AddTokenIdentifier(oldToken.DecodeIdentifier());
response = am.DoAllocateAs(ugi, Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken());
Org.Mockito.Mockito.Verify(spySecretMgr, Org.Mockito.Mockito.Never()).CreateAndGetAMRMToken
(Matchers.IsA <ApplicationAttemptId>());
// Do allocate again with the updated token and verify we do not
// receive a new token to use.
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
// Activate the next master key. Since there is new master key generated
// in AMRMTokenSecretManager. The AMRMToken will not get updated for AM
rm.GetRMContext().GetAMRMTokenSecretManager().ActivateNextMasterKey();
response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest
>());
NUnit.Framework.Assert.IsNull(response.GetAMRMToken());
rm.Stop();
}
public virtual void SetUp()
{
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
SetupPlanQueueConfiguration(conf);
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RmReservationSystemEnable, false);
rm = new MockRM(conf);
rm.Start();
}
public virtual void TestMaxAllocationAfterUpdateNodeResource()
{
int configuredMaxVCores = 20;
int configuredMaxMemory = 10 * 1024;
Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(configuredMaxMemory, configuredMaxVCores);
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 5);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(4096, 10);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource3 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(512, 1);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource4 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 2);
RMNode node1 = MockNodes.NewNodeInfo(0, resource1, 1, "127.0.0.2");
scheduler.Handle(new NodeAddedSchedulerEvent(node1));
RMNode node2 = MockNodes.NewNodeInfo(0, resource3, 2, "127.0.0.3");
scheduler.Handle(new NodeAddedSchedulerEvent(node2));
VerifyMaximumResourceCapability(resource1, scheduler);
// increase node1 resource
scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource2, 0));
VerifyMaximumResourceCapability(resource2, scheduler);
// decrease node1 resource
scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource1, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
// increase node2 resource
scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource4, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
// decrease node2 resource
scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource3, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
}
finally
{
rm.Stop();
}
}
public virtual void TestConcurrentAccess()
{
conf.Set(FairSchedulerConfiguration.AssignMultiple, "false");
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
string queueName = "root.queue1";
FSLeafQueue schedulable = scheduler.GetQueueManager().GetLeafQueue(queueName, true
);
ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1);
RMContext rmContext = resourceManager.GetRMContext();
FSAppAttempt app = new FSAppAttempt(scheduler, applicationAttemptId, "user1", schedulable
, null, rmContext);
// this needs to be in sync with the number of runnables declared below
int testThreads = 2;
IList <Runnable> runnables = new AList <Runnable>();
// add applications to modify the list
runnables.AddItem(new _Runnable_257(schedulable, app));
// iterate over the list a couple of times in a different thread
runnables.AddItem(new _Runnable_267(schedulable));
IList <Exception> exceptions = Sharpen.Collections.SynchronizedList(new AList <Exception
>());
ExecutorService threadPool = Executors.NewFixedThreadPool(testThreads);
try
{
CountDownLatch allExecutorThreadsReady = new CountDownLatch(testThreads);
CountDownLatch startBlocker = new CountDownLatch(1);
CountDownLatch allDone = new CountDownLatch(testThreads);
foreach (Runnable submittedTestRunnable in runnables)
{
threadPool.Submit(new _Runnable_287(allExecutorThreadsReady, startBlocker, submittedTestRunnable
, exceptions, allDone));
}
// wait until all threads are ready
allExecutorThreadsReady.Await();
// start all test runners
startBlocker.CountDown();
int testTimeout = 2;
NUnit.Framework.Assert.IsTrue("Timeout waiting for more than " + testTimeout + " seconds"
, allDone.Await(testTimeout, TimeUnit.Seconds));
}
catch (Exception ie)
{
exceptions.AddItem(ie);
}
finally
{
threadPool.ShutdownNow();
}
NUnit.Framework.Assert.IsTrue("Test failed with exception(s)" + exceptions, exceptions
.IsEmpty());
}
public virtual void TestMetricsInitializedOnRMInit()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
MockRM rm = new MockRM(conf);
QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics();
CheckApps(metrics, 0, 0, 0, 0, 0, 0, true);
MetricsAsserts.AssertGauge("ReservedContainers", 0, metrics);
}
private void StartResourceManager(float utilizationThreshold)
{
conf.SetFloat(FairSchedulerConfiguration.PreemptionThreshold, utilizationThreshold
);
resourceManager = new MockRM(conf);
resourceManager.Start();
NUnit.Framework.Assert.IsTrue(resourceManager.GetResourceScheduler() is TestFairSchedulerPreemption.StubbedFairScheduler
);
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
scheduler.SetClock(clock);
scheduler.updateInterval = 60 * 1000;
}
public virtual void TestAddAndRemoveAppFromFiFoScheduler()
{
Configuration conf = new Configuration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
MockRM rm = new MockRM(conf);
AbstractYarnScheduler <SchedulerApplicationAttempt, SchedulerNode> fs = (AbstractYarnScheduler
<SchedulerApplicationAttempt, SchedulerNode>)rm.GetResourceScheduler();
TestSchedulerUtils.VerifyAppAddedAndRemovedFromScheduler(fs.GetSchedulerApplications
(), fs, "queue");
}
public virtual void Setup()
{
mockClock = new FairSchedulerTestBase.MockClock();
conf = CreateConfiguration();
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
scheduler.SetClock(mockClock);
NUnit.Framework.Assert.IsTrue(scheduler.IsContinuousSchedulingEnabled());
NUnit.Framework.Assert.AreEqual(FairSchedulerConfiguration.DefaultContinuousSchedulingSleepMs
, scheduler.GetContinuousSchedulingSleepMs());
NUnit.Framework.Assert.AreEqual(mockClock, scheduler.GetClock());
}
public virtual void TestZKRootPathAcls()
{
HAServiceProtocol.StateChangeRequestInfo req = new HAServiceProtocol.StateChangeRequestInfo
(HAServiceProtocol.RequestSource.RequestByUser);
string rootPath = YarnConfiguration.DefaultZkRmStateStoreParentPath + "/" + ZKRMStateStore
.RootZnodeName;
// Start RM with HA enabled
Configuration conf = CreateHARMConf("rm1,rm2", "rm1", 1234);
conf.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false);
ResourceManager rm = new MockRM(conf);
rm.Start();
rm.GetRMContext().GetRMAdminService().TransitionToActive(req);
Stat stat = new Stat();
IList <ACL> acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries
(rootPath, stat);
NUnit.Framework.Assert.AreEqual(acls.Count, 2);
// CREATE and DELETE permissions for root node based on RM ID
VerifyZKACL("digest", "localhost", ZooDefs.Perms.Create | ZooDefs.Perms.Delete, acls
);
VerifyZKACL("world", "anyone", ZooDefs.Perms.All ^ (ZooDefs.Perms.Create | ZooDefs.Perms
.Delete), acls);
rm.Close();
// Now start RM with HA disabled. NoAuth Exception should not be thrown.
conf.SetBoolean(YarnConfiguration.RmHaEnabled, false);
rm = new MockRM(conf);
rm.Start();
rm.GetRMContext().GetRMAdminService().TransitionToActive(req);
acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries(rootPath
, stat);
NUnit.Framework.Assert.AreEqual(acls.Count, 1);
VerifyZKACL("world", "anyone", ZooDefs.Perms.All, acls);
rm.Close();
// Start RM with HA enabled.
conf.SetBoolean(YarnConfiguration.RmHaEnabled, true);
rm = new MockRM(conf);
rm.Start();
rm.GetRMContext().GetRMAdminService().TransitionToActive(req);
acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries(rootPath
, stat);
NUnit.Framework.Assert.AreEqual(acls.Count, 2);
VerifyZKACL("digest", "localhost", ZooDefs.Perms.Create | ZooDefs.Perms.Delete, acls
);
VerifyZKACL("world", "anyone", ZooDefs.Perms.All ^ (ZooDefs.Perms.Create | ZooDefs.Perms
.Delete), acls);
rm.Close();
}
/// <exception cref="System.Exception"/>
private void TestRMWritingMassiveHistory(MockRM rm)
{
rm.Start();
MockNM nm = rm.RegisterNode("127.0.0.1:1234", 1024 * 10100);
RMApp app = rm.SubmitApp(1024);
nm.NodeHeartbeat(true);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
int request = 10000;
am.Allocate("127.0.0.1", 1024, request, new AList <ContainerId>());
nm.NodeHeartbeat(true);
IList <Container> allocated = am.Allocate(new AList <ResourceRequest>(), new AList <
ContainerId>()).GetAllocatedContainers();
int waitCount = 0;
int allocatedSize = allocated.Count;
while (allocatedSize < request && waitCount++ < 200)
{
Sharpen.Thread.Sleep(300);
allocated = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers
();
allocatedSize += allocated.Count;
nm.NodeHeartbeat(true);
}
NUnit.Framework.Assert.AreEqual(request, allocatedSize);
am.UnregisterAppAttempt();
am.WaitForState(RMAppAttemptState.Finishing);
nm.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
NodeHeartbeatResponse resp = nm.NodeHeartbeat(true);
IList <ContainerId> cleaned = resp.GetContainersToCleanup();
int cleanedSize = cleaned.Count;
waitCount = 0;
while (cleanedSize < allocatedSize && waitCount++ < 200)
{
Sharpen.Thread.Sleep(300);
resp = nm.NodeHeartbeat(true);
cleaned = resp.GetContainersToCleanup();
cleanedSize += cleaned.Count;
}
NUnit.Framework.Assert.AreEqual(allocatedSize, cleanedSize);
rm.WaitForState(app.GetApplicationId(), RMAppState.Finished);
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestContainerAllocationWithSingleUserLimits()
{
RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
mgr.Init(conf);
// set node -> label
mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y"));
mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId
.NewInstance("h2", 0), ToSet("y")));
// inject node label manager
MockRM rm1 = new _MockRM_471(mgr, GetConfigurationWithDefaultQueueLabels(conf));
rm1.GetRMContext().SetNodeLabelManager(mgr);
rm1.Start();
MockNM nm1 = rm1.RegisterNode("h1:1234", 8000);
// label = x
rm1.RegisterNode("h2:1234", 8000);
// label = y
MockNM nm3 = rm1.RegisterNode("h3:1234", 8000);
// label = <empty>
// launch an app to queue a1 (label = x), and check all container will
// be allocated in h1
RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1");
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
// A has only 10% of x, so it can only allocate one container in label=empty
ContainerId containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 2);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty);
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
// Cannot allocate 2nd label=empty container
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 3);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty);
NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm3, containerId, RMContainerState
.Allocated, 10 * 1000));
// A has default user limit = 100, so it can use all resource in label = x
// We can allocate floor(8000 / 1024) = 7 containers
for (int id = 3; id <= 8; id++)
{
containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), id);
am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x");
NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState
.Allocated, 10 * 1000));
}
rm1.Close();
}
/// <exception cref="System.Exception"/>
public virtual void Test()
{
conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine("<queue name=\"queueA\">");
@out.WriteLine("<minResources>2048mb,0vcores</minResources>");
@out.WriteLine("</queue>");
@out.WriteLine("<queue name=\"queueB\">");
@out.WriteLine("<minResources>2048mb,0vcores</minResources>");
@out.WriteLine("</queue>");
@out.WriteLine("</allocations>");
@out.Close();
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
// Add one big node (only care about aggregate capacity)
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(4 * 1024, 4), 1,
"127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
scheduler.Update();
// Queue A wants 3 * 1024. Node update gives this all to A
CreateSchedulingRequest(3 * 1024, "queueA", "user1");
scheduler.Update();
NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);
scheduler.Handle(nodeEvent2);
// Queue B arrives and wants 1 * 1024
CreateSchedulingRequest(1 * 1024, "queueB", "user1");
scheduler.Update();
ICollection <FSLeafQueue> queues = scheduler.GetQueueManager().GetLeafQueues();
NUnit.Framework.Assert.AreEqual(3, queues.Count);
// Queue A should be above min share, B below.
FSLeafQueue queueA = scheduler.GetQueueManager().GetLeafQueue("queueA", false);
FSLeafQueue queueB = scheduler.GetQueueManager().GetLeafQueue("queueB", false);
NUnit.Framework.Assert.IsFalse(queueA.IsStarvedForMinShare());
NUnit.Framework.Assert.IsTrue(queueB.IsStarvedForMinShare());
// Node checks in again, should allocate for B
scheduler.Handle(nodeEvent2);
// Now B should have min share ( = demand here)
NUnit.Framework.Assert.IsFalse(queueB.IsStarvedForMinShare());
}
/// <exception cref="System.Exception"/>
private static void SetupAndStartRM()
{
Configuration rmconf = new Configuration();
rmconf.SetInt(YarnConfiguration.RmAmMaxAttempts, YarnConfiguration.DefaultRmAmMaxAttempts
);
rmconf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
rmconf.SetBoolean(YarnConfiguration.YarnAclEnable, true);
rmconf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"
);
rmconf.Set("yarn.resourcemanager.principal", spnegoPrincipal);
rmconf.Set("yarn.resourcemanager.keytab", spnegoKeytabFile.GetAbsolutePath());
rmconf.SetBoolean("mockrm.webapp.enabled", true);
UserGroupInformation.SetConfiguration(rmconf);
rm = new MockRM(rmconf);
rm.Start();
}
public virtual void TestMaximimumAllocationVCores()
{
int node1MaxVCores = 15;
int node2MaxVCores = 5;
int node3MaxVCores = 6;
int configuredMaxVCores = 10;
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 1000 * 1000
);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
TestMaximumAllocationVCoresHelper((AbstractYarnScheduler)rm.GetResourceScheduler(
), node1MaxVCores, node2MaxVCores, node3MaxVCores, configuredMaxVCores, configuredMaxVCores
, configuredMaxVCores, configuredMaxVCores, configuredMaxVCores, configuredMaxVCores
);
}
finally
{
rm.Stop();
}
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
rm = new MockRM(conf);
try
{
rm.Start();
TestMaximumAllocationVCoresHelper((AbstractYarnScheduler)rm.GetResourceScheduler(
), node1MaxVCores, node2MaxVCores, node3MaxVCores, configuredMaxVCores, configuredMaxVCores
, configuredMaxVCores, node2MaxVCores, node3MaxVCores, node2MaxVCores);
}
finally
{
rm.Stop();
}
}
public virtual void TestMaximimumAllocationMemory()
{
int node1MaxMemory = 15 * 1024;
int node2MaxMemory = 5 * 1024;
int node3MaxMemory = 6 * 1024;
int configuredMaxMemory = 10 * 1024;
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 1000 * 1000
);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
TestMaximumAllocationMemoryHelper((AbstractYarnScheduler)rm.GetResourceScheduler(
), node1MaxMemory, node2MaxMemory, node3MaxMemory, configuredMaxMemory, configuredMaxMemory
, configuredMaxMemory, configuredMaxMemory, configuredMaxMemory, configuredMaxMemory
);
}
finally
{
rm.Stop();
}
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
rm = new MockRM(conf);
try
{
rm.Start();
TestMaximumAllocationMemoryHelper((AbstractYarnScheduler)rm.GetResourceScheduler(
), node1MaxMemory, node2MaxMemory, node3MaxMemory, configuredMaxMemory, configuredMaxMemory
, configuredMaxMemory, node2MaxMemory, node3MaxMemory, node2MaxMemory);
}
finally
{
rm.Stop();
}
}
