public virtual void TestLogAggregationContextPassedIntoContainerToken() { MockRM rm1 = new MockRM(conf); rm1.Start(); MockNM nm1 = rm1.RegisterNode("127.0.0.1:1234", 8000); MockNM nm2 = rm1.RegisterNode("127.0.0.1:2345", 8000); // LogAggregationContext is set as null NUnit.Framework.Assert.IsNull(GetLogAggregationContextFromContainerToken(rm1, nm1 , null)); // create a not-null LogAggregationContext LogAggregationContext logAggregationContext = LogAggregationContext.NewInstance("includePattern" , "excludePattern", "rolledLogsIncludePattern", "rolledLogsExcludePattern"); LogAggregationContext returned = GetLogAggregationContextFromContainerToken(rm1, nm2, logAggregationContext); NUnit.Framework.Assert.AreEqual("includePattern", returned.GetIncludePattern()); NUnit.Framework.Assert.AreEqual("excludePattern", returned.GetExcludePattern()); NUnit.Framework.Assert.AreEqual("rolledLogsIncludePattern", returned.GetRolledLogsIncludePattern ()); NUnit.Framework.Assert.AreEqual("rolledLogsExcludePattern", returned.GetRolledLogsExcludePattern ()); rm1.Stop(); }
/// <exception cref="System.IO.IOException"/> private void CreateClusterWithQueuesAndOneNode(int mem, int vCores, string policy ) { PrintWriter @out = new PrintWriter(new FileWriter(AllocFile)); @out.WriteLine("<?xml version=\"1.0\"?>"); @out.WriteLine("<allocations>"); @out.WriteLine("<queue name=\"root\" >"); @out.WriteLine(" <queue name=\"parentA\" >"); @out.WriteLine(" <weight>8</weight>"); @out.WriteLine(" <queue name=\"childA1\" />"); @out.WriteLine(" <queue name=\"childA2\" />"); @out.WriteLine(" <queue name=\"childA3\" />"); @out.WriteLine(" <queue name=\"childA4\" />"); @out.WriteLine(" </queue>"); @out.WriteLine(" <queue name=\"parentB\" >"); @out.WriteLine(" <weight>1</weight>"); @out.WriteLine(" <queue name=\"childB1\" />"); @out.WriteLine(" <queue name=\"childB2\" />"); @out.WriteLine(" </queue>"); @out.WriteLine("</queue>"); @out.WriteLine("<defaultQueueSchedulingPolicy>" + policy + "</defaultQueueSchedulingPolicy>" ); @out.WriteLine("</allocations>"); @out.Close(); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(mem, vCores), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.Handle(nodeEvent1); }
public virtual void TestNormalContainerAllocationWhenDNSUnavailable() { MockRM rm1 = new MockRM(conf); rm1.Start(); MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000); RMApp app1 = rm1.SubmitApp(200); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); // request a container. am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>()); ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 2); rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated); // acquire the container. SecurityUtilTestHelper.SetTokenServiceUseIp(true); IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers(); // not able to fetch the container; NUnit.Framework.Assert.AreEqual(0, containers.Count); SecurityUtilTestHelper.SetTokenServiceUseIp(false); containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()) .GetAllocatedContainers(); // should be able to fetch the container; NUnit.Framework.Assert.AreEqual(1, containers.Count); }
public virtual void SetUp() { dispatcher = new DrainDispatcher(); this.rm = new _MockRM_58(this); rm.Start(); amService = rm.GetApplicationMasterService(); }
// Test regular RM restart/failover, new RM should not count // AM failure towards the max-retry-account and should be able to // re-launch the AM. /// <exception cref="System.Exception"/> public virtual void TestRMRestartOrFailoverNotCountedForAMFailures() { YarnConfiguration conf = new YarnConfiguration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler )); conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true); conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false); conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName); // explicitly set max-am-retry count as 1. conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); MockRM rm1 = new MockRM(conf, memStore); rm1.Start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService()); nm1.RegisterNode(); RMApp app1 = rm1.SubmitApp(200); // AM should be restarted even though max-am-attempt is 1. MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt1).MayBeLastAttempt()); // Restart rm. MockRM rm2 = new MockRM(conf, memStore); rm2.Start(); ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId ()]; // re-register the NM nm1.SetResourceTrackerService(rm2.GetResourceTrackerService()); NMContainerStatus status = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NMContainerStatus >(); status.SetContainerExitStatus(ContainerExitStatus.KilledByResourcemanager); status.SetContainerId(attempt1.GetMasterContainer().GetId()); status.SetContainerState(ContainerState.Complete); status.SetDiagnostics(string.Empty); nm1.RegisterNode(Sharpen.Collections.SingletonList(status), null); rm2.WaitForState(attempt1.GetAppAttemptId(), RMAppAttemptState.Failed); NUnit.Framework.Assert.AreEqual(ContainerExitStatus.KilledByResourcemanager, appState .GetAttempt(am1.GetApplicationAttemptId()).GetAMContainerExitStatus()); // Will automatically start a new AppAttempt in rm2 rm2.WaitForState(app1.GetApplicationId(), RMAppState.Accepted); MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1); MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2); RMAppAttempt attempt3 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt (); NUnit.Framework.Assert.IsTrue(attempt3.ShouldCountTowardsMaxAttemptRetry()); NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt( am2.GetApplicationAttemptId()).GetAMContainerExitStatus()); rm1.Stop(); rm2.Stop(); }
public virtual void TestContainerTokenGeneratedOnPullRequest() { MockRM rm1 = new MockRM(conf); rm1.Start(); MockNM nm1 = rm1.RegisterNode("127.0.0.1:1234", 8000); RMApp app1 = rm1.SubmitApp(200); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); // request a container. am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>()); ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 2); rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated); RMContainer container = rm1.GetResourceScheduler().GetRMContainer(containerId2); // no container token is generated. NUnit.Framework.Assert.AreEqual(containerId2, container.GetContainerId()); NUnit.Framework.Assert.IsNull(container.GetContainer().GetContainerToken()); // acquire the container. IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers(); NUnit.Framework.Assert.AreEqual(containerId2, containers[0].GetId()); // container token is generated. NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken()); rm1.Stop(); }
public virtual void TestUpdateMaxAllocationUsesTotal() { int configuredMaxVCores = 20; int configuredMaxMemory = 10 * 1024; Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(configuredMaxMemory, configuredMaxVCores); ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores ); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); MockRM rm = new MockRM(conf); try { rm.Start(); AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler( ); Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(0, 0); Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1024, 5); Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2048, 10); SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>(); Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo" , 8080)); Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource ); Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1); SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>(); Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar" , 8081)); Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource ); Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); scheduler.nodes = new Dictionary <NodeId, SchedulerNode>(); scheduler.nodes[mockNode1.GetNodeID()] = mockNode1; scheduler.UpdateMaximumAllocation(mockNode1, true); VerifyMaximumResourceCapability(fullResource1, scheduler); scheduler.nodes[mockNode2.GetNodeID()] = mockNode2; scheduler.UpdateMaximumAllocation(mockNode2, true); VerifyMaximumResourceCapability(fullResource2, scheduler); Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID()); scheduler.UpdateMaximumAllocation(mockNode2, false); VerifyMaximumResourceCapability(fullResource1, scheduler); Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID()); scheduler.UpdateMaximumAllocation(mockNode1, false); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); } finally { rm.Stop(); } }
public virtual void TestExistenceOfResourceRequestInRMContainer() { Configuration conf = new Configuration(); MockRM rm1 = new MockRM(conf); rm1.Start(); MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000); RMApp app1 = rm1.SubmitApp(1024); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); ResourceScheduler scheduler = rm1.GetResourceScheduler(); // request a container. am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>()); ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 2); rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated); // Verify whether list of ResourceRequest is present in RMContainer // while moving to ALLOCATED state NUnit.Framework.Assert.IsNotNull(scheduler.GetRMContainer(containerId2).GetResourceRequests ()); // Allocate container am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers (); rm1.WaitForState(nm1, containerId2, RMContainerState.Acquired); // After RMContainer moving to ACQUIRED state, list of ResourceRequest will // be empty NUnit.Framework.Assert.IsNull(scheduler.GetRMContainer(containerId2).GetResourceRequests ()); }
/// <exception cref="System.Exception"/> private void TestRMWritingMassiveHistory(bool isFS) { // 1. Show RM can run with writing history data // 2. Test additional workload of processing history events YarnConfiguration conf = new YarnConfiguration(); if (isFS) { conf.SetBoolean(FairSchedulerConfiguration.AssignMultiple, true); conf.Set("yarn.resourcemanager.scheduler.class", typeof(FairScheduler).FullName); } else { conf.Set("yarn.resourcemanager.scheduler.class", typeof(CapacityScheduler).FullName ); } // don't process history events MockRM rm = new _MockRM_399(conf); long startTime1 = Runtime.CurrentTimeMillis(); TestRMWritingMassiveHistory(rm); long finishTime1 = Runtime.CurrentTimeMillis(); long elapsedTime1 = finishTime1 - startTime1; rm = new MockRM(conf); long startTime2 = Runtime.CurrentTimeMillis(); TestRMWritingMassiveHistory(rm); long finishTime2 = Runtime.CurrentTimeMillis(); long elapsedTime2 = finishTime2 - startTime2; // No more than 10% additional workload // Should be much less, but computation time is fluctuated NUnit.Framework.Assert.IsTrue(elapsedTime2 - elapsedTime1 < elapsedTime1 / 10); }
// This is to test fetching AM container will be retried, if AM container is // not fetchable since DNS is unavailable causing container token/NMtoken // creation failure. /// <exception cref="System.Exception"/> public virtual void TestAMContainerAllocationWhenDNSUnavailable() { MockRM rm1 = new _MockRM_303(this, conf); rm1.Start(); MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000); SecurityUtilTestHelper.SetTokenServiceUseIp(true); RMApp app1 = rm1.SubmitApp(200); RMAppAttempt attempt = app1.GetCurrentAppAttempt(); nm1.NodeHeartbeat(true); // fetching am container will fail, keep retrying 5 times. while (numRetries <= 5) { nm1.NodeHeartbeat(true); Sharpen.Thread.Sleep(1000); NUnit.Framework.Assert.AreEqual(RMAppAttemptState.Scheduled, attempt.GetAppAttemptState ()); System.Console.Out.WriteLine("Waiting for am container to be allocated."); } SecurityUtilTestHelper.SetTokenServiceUseIp(false); rm1.WaitForState(attempt.GetAppAttemptId(), RMAppAttemptState.Allocated); MockRM.LaunchAndRegisterAM(app1, rm1, nm1); }
/// <exception cref="System.Exception"/> private LogAggregationContext GetLogAggregationContextFromContainerToken(MockRM rm1 , MockNM nm1, LogAggregationContext logAggregationContext) { RMApp app2 = rm1.SubmitApp(200, logAggregationContext); MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm1); nm1.NodeHeartbeat(true); // request a container. am2.Allocate("127.0.0.1", 512, 1, new AList <ContainerId>()); ContainerId containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId( ), 2); rm1.WaitForState(nm1, containerId, RMContainerState.Allocated); // acquire the container. IList <Container> containers = am2.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers(); NUnit.Framework.Assert.AreEqual(containerId, containers[0].GetId()); // container token is generated. NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken()); ContainerTokenIdentifier token = BuilderUtils.NewContainerTokenIdentifier(containers [0].GetContainerToken()); return(token.GetLogAggregationContext()); }
/// <exception cref="System.Exception"/> public virtual void TestContainerAllocateWithLabels() { // set node -> label mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y")); mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId .NewInstance("h2", 0), ToSet("y"))); // inject node label manager MockRM rm1 = new _MockRM_638(this, GetConfigurationWithQueueLabels(conf)); rm1.GetRMContext().SetNodeLabelManager(mgr); rm1.Start(); MockNM nm1 = rm1.RegisterNode("h1:1234", 8000); // label = x MockNM nm2 = rm1.RegisterNode("h2:1234", 8000); // label = y MockNM nm3 = rm1.RegisterNode("h3:1234", 8000); // label = <empty> ContainerId containerId; // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1"); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm3); // request a container. am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x"); containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 2); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState .Allocated, 10 * 1000)); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState .Allocated, 10 * 1000)); CheckTaskContainersHost(am1.GetApplicationAttemptId(), containerId, rm1, "h1"); // launch an app to queue b1 (label = y), and check all container will // be allocated in h2 RMApp app2 = rm1.SubmitApp(200, "app", "user", null, "b1"); MockAM am2 = MockRM.LaunchAndRegisterAM(app2, rm1, nm3); // request a container. am2.Allocate("*", 1024, 1, new AList <ContainerId>(), "y"); containerId = ContainerId.NewContainerId(am2.GetApplicationAttemptId(), 2); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm1, containerId, RMContainerState .Allocated, 10 * 1000)); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm2, containerId, RMContainerState .Allocated, 10 * 1000)); CheckTaskContainersHost(am2.GetApplicationAttemptId(), containerId, rm1, "h2"); // launch an app to queue c1 (label = ""), and check all container will // be allocated in h3 RMApp app3 = rm1.SubmitApp(200, "app", "user", null, "c1"); MockAM am3 = MockRM.LaunchAndRegisterAM(app3, rm1, nm3); // request a container. am3.Allocate("*", 1024, 1, new AList <ContainerId>()); containerId = ContainerId.NewContainerId(am3.GetApplicationAttemptId(), 2); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm2, containerId, RMContainerState .Allocated, 10 * 1000)); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState .Allocated, 10 * 1000)); CheckTaskContainersHost(am3.GetApplicationAttemptId(), containerId, rm1, "h3"); rm1.Close(); }
// Test RM restarts after AM container is preempted, new RM should not count // AM preemption failure towards the max-retry-account and should be able to // re-launch the AM. /// <exception cref="System.Exception"/> public virtual void TestPreemptedAMRestartOnRMRestart() { YarnConfiguration conf = new YarnConfiguration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler )); conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true); conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false); conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName); // explicitly set max-am-retry count as 1. conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); MockRM rm1 = new MockRM(conf, memStore); rm1.Start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService()); nm1.RegisterNode(); RMApp app1 = rm1.SubmitApp(200); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler(); ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId( ), 1); // Forcibly preempt the am container; scheduler.KillContainer(scheduler.GetRMContainer(amContainer)); am1.WaitForState(RMAppAttemptState.Failed); NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry()); rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted); // state store has 1 attempt stored. ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId ()]; NUnit.Framework.Assert.AreEqual(1, appState.GetAttemptCount()); // attempt stored has the preempted container exit status. NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Preempted, appState.GetAttempt (am1.GetApplicationAttemptId()).GetAMContainerExitStatus()); // Restart rm. MockRM rm2 = new MockRM(conf, memStore); nm1.SetResourceTrackerService(rm2.GetResourceTrackerService()); nm1.RegisterNode(); rm2.Start(); // Restarted RM should re-launch the am. MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1); MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2); RMAppAttempt attempt2 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt (); NUnit.Framework.Assert.IsTrue(attempt2.ShouldCountTowardsMaxAttemptRetry()); NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt( am2.GetApplicationAttemptId()).GetAMContainerExitStatus()); rm1.Stop(); rm2.Stop(); }
public virtual void Setup() { Configuration conf = CreateConfiguration(); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); }
private void CheckUsedResource(MockRM rm, string queueName, int memory, string label ) { CapacityScheduler scheduler = (CapacityScheduler)rm.GetResourceScheduler(); CSQueue queue = scheduler.GetQueue(queueName); NUnit.Framework.Assert.AreEqual(memory, queue.GetQueueResourceUsage().GetUsed(label ).GetMemory()); }
// To avoid using cached client /// <exception cref="System.Exception"/> public virtual void TestAMRMMasterKeysUpdate() { AtomicReference <AMRMTokenSecretManager> spySecretMgrRef = new AtomicReference <AMRMTokenSecretManager >(); MockRM rm = new _MockRM_349(this, spySecretMgrRef, conf); // Skip the login. rm.Start(); MockNM nm = rm.RegisterNode("127.0.0.1:1234", 8000); RMApp app = rm.SubmitApp(200); MockAM am = MockRM.LaunchAndRegisterAM(app, rm, nm); AMRMTokenSecretManager spySecretMgr = spySecretMgrRef.Get(); // Do allocate. Should not update AMRMToken AllocateResponse response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest>()); NUnit.Framework.Assert.IsNull(response.GetAMRMToken()); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> oldToken = rm.GetRMContext ().GetRMApps()[app.GetApplicationId()].GetRMAppAttempt(am.GetApplicationAttemptId ()).GetAMRMToken(); // roll over the master key // Do allocate again. the AM should get the latest AMRMToken rm.GetRMContext().GetAMRMTokenSecretManager().RollMasterKey(); response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >()); NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken()); Org.Apache.Hadoop.Security.Token.Token <AMRMTokenIdentifier> amrmToken = ConverterUtils .ConvertFromYarn(response.GetAMRMToken(), new Text(response.GetAMRMToken().GetService ())); NUnit.Framework.Assert.AreEqual(amrmToken.DecodeIdentifier().GetKeyId(), rm.GetRMContext ().GetAMRMTokenSecretManager().GetMasterKey().GetMasterKey().GetKeyId()); // Do allocate again with the same old token and verify the RM sends // back the last generated token instead of generating it again. Org.Mockito.Mockito.Reset(spySecretMgr); UserGroupInformation ugi = UserGroupInformation.CreateUserForTesting(am.GetApplicationAttemptId ().ToString(), new string[0]); ugi.AddTokenIdentifier(oldToken.DecodeIdentifier()); response = am.DoAllocateAs(ugi, Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >()); NUnit.Framework.Assert.IsNotNull(response.GetAMRMToken()); Org.Mockito.Mockito.Verify(spySecretMgr, Org.Mockito.Mockito.Never()).CreateAndGetAMRMToken (Matchers.IsA <ApplicationAttemptId>()); // Do allocate again with the updated token and verify we do not // receive a new token to use. response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >()); NUnit.Framework.Assert.IsNull(response.GetAMRMToken()); // Activate the next master key. Since there is new master key generated // in AMRMTokenSecretManager. The AMRMToken will not get updated for AM rm.GetRMContext().GetAMRMTokenSecretManager().ActivateNextMasterKey(); response = am.Allocate(Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <AllocateRequest >()); NUnit.Framework.Assert.IsNull(response.GetAMRMToken()); rm.Stop(); }
public virtual void SetUp() { CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration(); SetupPlanQueueConfiguration(conf); conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler )); conf.SetBoolean(YarnConfiguration.RmReservationSystemEnable, false); rm = new MockRM(conf); rm.Start(); }
public virtual void TestMaxAllocationAfterUpdateNodeResource() { int configuredMaxVCores = 20; int configuredMaxMemory = 10 * 1024; Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(configuredMaxMemory, configuredMaxVCores); ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores ); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); MockRM rm = new MockRM(conf); try { rm.Start(); AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler( ); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2048, 5); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(4096, 10); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource3 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(512, 1); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource4 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1024, 2); RMNode node1 = MockNodes.NewNodeInfo(0, resource1, 1, "127.0.0.2"); scheduler.Handle(new NodeAddedSchedulerEvent(node1)); RMNode node2 = MockNodes.NewNodeInfo(0, resource3, 2, "127.0.0.3"); scheduler.Handle(new NodeAddedSchedulerEvent(node2)); VerifyMaximumResourceCapability(resource1, scheduler); // increase node1 resource scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource2, 0)); VerifyMaximumResourceCapability(resource2, scheduler); // decrease node1 resource scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource1, 0)); VerifyMaximumResourceCapability(resource1, scheduler); // increase node2 resource scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource4, 0)); VerifyMaximumResourceCapability(resource1, scheduler); // decrease node2 resource scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource3, 0)); VerifyMaximumResourceCapability(resource1, scheduler); } finally { rm.Stop(); } }
public virtual void TestConcurrentAccess() { conf.Set(FairSchedulerConfiguration.AssignMultiple, "false"); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); string queueName = "root.queue1"; FSLeafQueue schedulable = scheduler.GetQueueManager().GetLeafQueue(queueName, true ); ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1); RMContext rmContext = resourceManager.GetRMContext(); FSAppAttempt app = new FSAppAttempt(scheduler, applicationAttemptId, "user1", schedulable , null, rmContext); // this needs to be in sync with the number of runnables declared below int testThreads = 2; IList <Runnable> runnables = new AList <Runnable>(); // add applications to modify the list runnables.AddItem(new _Runnable_257(schedulable, app)); // iterate over the list a couple of times in a different thread runnables.AddItem(new _Runnable_267(schedulable)); IList <Exception> exceptions = Sharpen.Collections.SynchronizedList(new AList <Exception >()); ExecutorService threadPool = Executors.NewFixedThreadPool(testThreads); try { CountDownLatch allExecutorThreadsReady = new CountDownLatch(testThreads); CountDownLatch startBlocker = new CountDownLatch(1); CountDownLatch allDone = new CountDownLatch(testThreads); foreach (Runnable submittedTestRunnable in runnables) { threadPool.Submit(new _Runnable_287(allExecutorThreadsReady, startBlocker, submittedTestRunnable , exceptions, allDone)); } // wait until all threads are ready allExecutorThreadsReady.Await(); // start all test runners startBlocker.CountDown(); int testTimeout = 2; NUnit.Framework.Assert.IsTrue("Timeout waiting for more than " + testTimeout + " seconds" , allDone.Await(testTimeout, TimeUnit.Seconds)); } catch (Exception ie) { exceptions.AddItem(ie); } finally { threadPool.ShutdownNow(); } NUnit.Framework.Assert.IsTrue("Test failed with exception(s)" + exceptions, exceptions .IsEmpty()); }
public virtual void TestMetricsInitializedOnRMInit() { YarnConfiguration conf = new YarnConfiguration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics(); CheckApps(metrics, 0, 0, 0, 0, 0, 0, true); MetricsAsserts.AssertGauge("ReservedContainers", 0, metrics); }
private void StartResourceManager(float utilizationThreshold) { conf.SetFloat(FairSchedulerConfiguration.PreemptionThreshold, utilizationThreshold ); resourceManager = new MockRM(conf); resourceManager.Start(); NUnit.Framework.Assert.IsTrue(resourceManager.GetResourceScheduler() is TestFairSchedulerPreemption.StubbedFairScheduler ); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); scheduler.SetClock(clock); scheduler.updateInterval = 60 * 1000; }
public virtual void TestAddAndRemoveAppFromFiFoScheduler() { Configuration conf = new Configuration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); AbstractYarnScheduler <SchedulerApplicationAttempt, SchedulerNode> fs = (AbstractYarnScheduler <SchedulerApplicationAttempt, SchedulerNode>)rm.GetResourceScheduler(); TestSchedulerUtils.VerifyAppAddedAndRemovedFromScheduler(fs.GetSchedulerApplications (), fs, "queue"); }
public virtual void Setup() { mockClock = new FairSchedulerTestBase.MockClock(); conf = CreateConfiguration(); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); scheduler.SetClock(mockClock); NUnit.Framework.Assert.IsTrue(scheduler.IsContinuousSchedulingEnabled()); NUnit.Framework.Assert.AreEqual(FairSchedulerConfiguration.DefaultContinuousSchedulingSleepMs , scheduler.GetContinuousSchedulingSleepMs()); NUnit.Framework.Assert.AreEqual(mockClock, scheduler.GetClock()); }
public virtual void TestZKRootPathAcls() { HAServiceProtocol.StateChangeRequestInfo req = new HAServiceProtocol.StateChangeRequestInfo (HAServiceProtocol.RequestSource.RequestByUser); string rootPath = YarnConfiguration.DefaultZkRmStateStoreParentPath + "/" + ZKRMStateStore .RootZnodeName; // Start RM with HA enabled Configuration conf = CreateHARMConf("rm1,rm2", "rm1", 1234); conf.SetBoolean(YarnConfiguration.AutoFailoverEnabled, false); ResourceManager rm = new MockRM(conf); rm.Start(); rm.GetRMContext().GetRMAdminService().TransitionToActive(req); Stat stat = new Stat(); IList <ACL> acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries (rootPath, stat); NUnit.Framework.Assert.AreEqual(acls.Count, 2); // CREATE and DELETE permissions for root node based on RM ID VerifyZKACL("digest", "localhost", ZooDefs.Perms.Create | ZooDefs.Perms.Delete, acls ); VerifyZKACL("world", "anyone", ZooDefs.Perms.All ^ (ZooDefs.Perms.Create | ZooDefs.Perms .Delete), acls); rm.Close(); // Now start RM with HA disabled. NoAuth Exception should not be thrown. conf.SetBoolean(YarnConfiguration.RmHaEnabled, false); rm = new MockRM(conf); rm.Start(); rm.GetRMContext().GetRMAdminService().TransitionToActive(req); acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries(rootPath , stat); NUnit.Framework.Assert.AreEqual(acls.Count, 1); VerifyZKACL("world", "anyone", ZooDefs.Perms.All, acls); rm.Close(); // Start RM with HA enabled. conf.SetBoolean(YarnConfiguration.RmHaEnabled, true); rm = new MockRM(conf); rm.Start(); rm.GetRMContext().GetRMAdminService().TransitionToActive(req); acls = ((ZKRMStateStore)rm.GetRMContext().GetStateStore()).GetACLWithRetries(rootPath , stat); NUnit.Framework.Assert.AreEqual(acls.Count, 2); VerifyZKACL("digest", "localhost", ZooDefs.Perms.Create | ZooDefs.Perms.Delete, acls ); VerifyZKACL("world", "anyone", ZooDefs.Perms.All ^ (ZooDefs.Perms.Create | ZooDefs.Perms .Delete), acls); rm.Close(); }
/// <exception cref="System.Exception"/> private void TestRMWritingMassiveHistory(MockRM rm) { rm.Start(); MockNM nm = rm.RegisterNode("127.0.0.1:1234", 1024 * 10100); RMApp app = rm.SubmitApp(1024); nm.NodeHeartbeat(true); RMAppAttempt attempt = app.GetCurrentAppAttempt(); MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId()); am.RegisterAppAttempt(); int request = 10000; am.Allocate("127.0.0.1", 1024, request, new AList <ContainerId>()); nm.NodeHeartbeat(true); IList <Container> allocated = am.Allocate(new AList <ResourceRequest>(), new AList < ContainerId>()).GetAllocatedContainers(); int waitCount = 0; int allocatedSize = allocated.Count; while (allocatedSize < request && waitCount++ < 200) { Sharpen.Thread.Sleep(300); allocated = am.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers (); allocatedSize += allocated.Count; nm.NodeHeartbeat(true); } NUnit.Framework.Assert.AreEqual(request, allocatedSize); am.UnregisterAppAttempt(); am.WaitForState(RMAppAttemptState.Finishing); nm.NodeHeartbeat(am.GetApplicationAttemptId(), 1, ContainerState.Complete); am.WaitForState(RMAppAttemptState.Finished); NodeHeartbeatResponse resp = nm.NodeHeartbeat(true); IList <ContainerId> cleaned = resp.GetContainersToCleanup(); int cleanedSize = cleaned.Count; waitCount = 0; while (cleanedSize < allocatedSize && waitCount++ < 200) { Sharpen.Thread.Sleep(300); resp = nm.NodeHeartbeat(true); cleaned = resp.GetContainersToCleanup(); cleanedSize += cleaned.Count; } NUnit.Framework.Assert.AreEqual(allocatedSize, cleanedSize); rm.WaitForState(app.GetApplicationId(), RMAppState.Finished); rm.Stop(); }
/// <exception cref="System.Exception"/> public virtual void TestContainerAllocationWithSingleUserLimits() { RMNodeLabelsManager mgr = new NullRMNodeLabelsManager(); mgr.Init(conf); // set node -> label mgr.AddToCluserNodeLabels(ImmutableSet.Of("x", "y")); mgr.AddLabelsToNode(ImmutableMap.Of(NodeId.NewInstance("h1", 0), ToSet("x"), NodeId .NewInstance("h2", 0), ToSet("y"))); // inject node label manager MockRM rm1 = new _MockRM_471(mgr, GetConfigurationWithDefaultQueueLabels(conf)); rm1.GetRMContext().SetNodeLabelManager(mgr); rm1.Start(); MockNM nm1 = rm1.RegisterNode("h1:1234", 8000); // label = x rm1.RegisterNode("h2:1234", 8000); // label = y MockNM nm3 = rm1.RegisterNode("h3:1234", 8000); // label = <empty> // launch an app to queue a1 (label = x), and check all container will // be allocated in h1 RMApp app1 = rm1.SubmitApp(200, "app", "user", null, "a1"); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); // A has only 10% of x, so it can only allocate one container in label=empty ContainerId containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId( ), 2); am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm3, containerId, RMContainerState .Allocated, 10 * 1000)); // Cannot allocate 2nd label=empty container containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), 3); am1.Allocate("*", 1024, 1, new AList <ContainerId>(), string.Empty); NUnit.Framework.Assert.IsFalse(rm1.WaitForState(nm3, containerId, RMContainerState .Allocated, 10 * 1000)); // A has default user limit = 100, so it can use all resource in label = x // We can allocate floor(8000 / 1024) = 7 containers for (int id = 3; id <= 8; id++) { containerId = ContainerId.NewContainerId(am1.GetApplicationAttemptId(), id); am1.Allocate("*", 1024, 1, new AList <ContainerId>(), "x"); NUnit.Framework.Assert.IsTrue(rm1.WaitForState(nm1, containerId, RMContainerState .Allocated, 10 * 1000)); } rm1.Close(); }
/// <exception cref="System.Exception"/> public virtual void Test() { conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile); PrintWriter @out = new PrintWriter(new FileWriter(AllocFile)); @out.WriteLine("<?xml version=\"1.0\"?>"); @out.WriteLine("<allocations>"); @out.WriteLine("<queue name=\"queueA\">"); @out.WriteLine("<minResources>2048mb,0vcores</minResources>"); @out.WriteLine("</queue>"); @out.WriteLine("<queue name=\"queueB\">"); @out.WriteLine("<minResources>2048mb,0vcores</minResources>"); @out.WriteLine("</queue>"); @out.WriteLine("</allocations>"); @out.Close(); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(4 * 1024, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.Handle(nodeEvent1); scheduler.Update(); // Queue A wants 3 * 1024. Node update gives this all to A CreateSchedulingRequest(3 * 1024, "queueA", "user1"); scheduler.Update(); NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1); scheduler.Handle(nodeEvent2); // Queue B arrives and wants 1 * 1024 CreateSchedulingRequest(1 * 1024, "queueB", "user1"); scheduler.Update(); ICollection <FSLeafQueue> queues = scheduler.GetQueueManager().GetLeafQueues(); NUnit.Framework.Assert.AreEqual(3, queues.Count); // Queue A should be above min share, B below. FSLeafQueue queueA = scheduler.GetQueueManager().GetLeafQueue("queueA", false); FSLeafQueue queueB = scheduler.GetQueueManager().GetLeafQueue("queueB", false); NUnit.Framework.Assert.IsFalse(queueA.IsStarvedForMinShare()); NUnit.Framework.Assert.IsTrue(queueB.IsStarvedForMinShare()); // Node checks in again, should allocate for B scheduler.Handle(nodeEvent2); // Now B should have min share ( = demand here) NUnit.Framework.Assert.IsFalse(queueB.IsStarvedForMinShare()); }
/// <exception cref="System.Exception"/> private static void SetupAndStartRM() { Configuration rmconf = new Configuration(); rmconf.SetInt(YarnConfiguration.RmAmMaxAttempts, YarnConfiguration.DefaultRmAmMaxAttempts ); rmconf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); rmconf.SetBoolean(YarnConfiguration.YarnAclEnable, true); rmconf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos" ); rmconf.Set("yarn.resourcemanager.principal", spnegoPrincipal); rmconf.Set("yarn.resourcemanager.keytab", spnegoKeytabFile.GetAbsolutePath()); rmconf.SetBoolean("mockrm.webapp.enabled", true); UserGroupInformation.SetConfiguration(rmconf); rm = new MockRM(rmconf); rm.Start(); }
public virtual void TestMaximimumAllocationVCores() { int node1MaxVCores = 15; int node2MaxVCores = 5; int node3MaxVCores = 6; int configuredMaxVCores = 10; ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 1000 * 1000 ); MockRM rm = new MockRM(conf); try { rm.Start(); TestMaximumAllocationVCoresHelper((AbstractYarnScheduler)rm.GetResourceScheduler( ), node1MaxVCores, node2MaxVCores, node3MaxVCores, configuredMaxVCores, configuredMaxVCores , configuredMaxVCores, configuredMaxVCores, configuredMaxVCores, configuredMaxVCores ); } finally { rm.Stop(); } conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); rm = new MockRM(conf); try { rm.Start(); TestMaximumAllocationVCoresHelper((AbstractYarnScheduler)rm.GetResourceScheduler( ), node1MaxVCores, node2MaxVCores, node3MaxVCores, configuredMaxVCores, configuredMaxVCores , configuredMaxVCores, node2MaxVCores, node3MaxVCores, node2MaxVCores); } finally { rm.Stop(); } }
public virtual void TestMaximimumAllocationMemory() { int node1MaxMemory = 15 * 1024; int node2MaxMemory = 5 * 1024; int node3MaxMemory = 6 * 1024; int configuredMaxMemory = 10 * 1024; ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 1000 * 1000 ); MockRM rm = new MockRM(conf); try { rm.Start(); TestMaximumAllocationMemoryHelper((AbstractYarnScheduler)rm.GetResourceScheduler( ), node1MaxMemory, node2MaxMemory, node3MaxMemory, configuredMaxMemory, configuredMaxMemory , configuredMaxMemory, configuredMaxMemory, configuredMaxMemory, configuredMaxMemory ); } finally { rm.Stop(); } conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); rm = new MockRM(conf); try { rm.Start(); TestMaximumAllocationMemoryHelper((AbstractYarnScheduler)rm.GetResourceScheduler( ), node1MaxMemory, node2MaxMemory, node3MaxMemory, configuredMaxMemory, configuredMaxMemory , configuredMaxMemory, node2MaxMemory, node3MaxMemory, node2MaxMemory); } finally { rm.Stop(); } }