private void StartResourceManager(float utilizationThreshold) { conf.SetFloat(FairSchedulerConfiguration.PreemptionThreshold, utilizationThreshold ); resourceManager = new MockRM(conf); resourceManager.Start(); NUnit.Framework.Assert.IsTrue(resourceManager.GetResourceScheduler() is TestFairSchedulerPreemption.StubbedFairScheduler ); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); scheduler.SetClock(clock); scheduler.updateInterval = 60 * 1000; }
public virtual void TestContainerTokenGeneratedOnPullRequest() { MockRM rm1 = new MockRM(conf); rm1.Start(); MockNM nm1 = rm1.RegisterNode("127.0.0.1:1234", 8000); RMApp app1 = rm1.SubmitApp(200); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); // request a container. am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>()); ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 2); rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated); RMContainer container = rm1.GetResourceScheduler().GetRMContainer(containerId2); // no container token is generated. NUnit.Framework.Assert.AreEqual(containerId2, container.GetContainerId()); NUnit.Framework.Assert.IsNull(container.GetContainer().GetContainerToken()); // acquire the container. IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers(); NUnit.Framework.Assert.AreEqual(containerId2, containers[0].GetId()); // container token is generated. NUnit.Framework.Assert.IsNotNull(containers[0].GetContainerToken()); rm1.Stop(); }
public virtual void TestExistenceOfResourceRequestInRMContainer() { Configuration conf = new Configuration(); MockRM rm1 = new MockRM(conf); rm1.Start(); MockNM nm1 = rm1.RegisterNode("unknownhost:1234", 8000); RMApp app1 = rm1.SubmitApp(1024); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); ResourceScheduler scheduler = rm1.GetResourceScheduler(); // request a container. am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>()); ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 2); rm1.WaitForState(nm1, containerId2, RMContainerState.Allocated); // Verify whether list of ResourceRequest is present in RMContainer // while moving to ALLOCATED state NUnit.Framework.Assert.IsNotNull(scheduler.GetRMContainer(containerId2).GetResourceRequests ()); // Allocate container am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers (); rm1.WaitForState(nm1, containerId2, RMContainerState.Acquired); // After RMContainer moving to ACQUIRED state, list of ResourceRequest will // be empty NUnit.Framework.Assert.IsNull(scheduler.GetRMContainer(containerId2).GetResourceRequests ()); }
public virtual void TestUpdateMaxAllocationUsesTotal() { int configuredMaxVCores = 20; int configuredMaxMemory = 10 * 1024; Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(configuredMaxMemory, configuredMaxVCores); ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores ); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); MockRM rm = new MockRM(conf); try { rm.Start(); AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler( ); Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(0, 0); Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1024, 5); Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2048, 10); SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>(); Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo" , 8080)); Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource ); Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1); SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>(); Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar" , 8081)); Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource ); Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); scheduler.nodes = new Dictionary <NodeId, SchedulerNode>(); scheduler.nodes[mockNode1.GetNodeID()] = mockNode1; scheduler.UpdateMaximumAllocation(mockNode1, true); VerifyMaximumResourceCapability(fullResource1, scheduler); scheduler.nodes[mockNode2.GetNodeID()] = mockNode2; scheduler.UpdateMaximumAllocation(mockNode2, true); VerifyMaximumResourceCapability(fullResource2, scheduler); Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID()); scheduler.UpdateMaximumAllocation(mockNode2, false); VerifyMaximumResourceCapability(fullResource1, scheduler); Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID()); scheduler.UpdateMaximumAllocation(mockNode1, false); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); } finally { rm.Stop(); } }
/// <exception cref="System.IO.IOException"/> private void CreateClusterWithQueuesAndOneNode(int mem, int vCores, string policy ) { PrintWriter @out = new PrintWriter(new FileWriter(AllocFile)); @out.WriteLine("<?xml version=\"1.0\"?>"); @out.WriteLine("<allocations>"); @out.WriteLine("<queue name=\"root\" >"); @out.WriteLine(" <queue name=\"parentA\" >"); @out.WriteLine(" <weight>8</weight>"); @out.WriteLine(" <queue name=\"childA1\" />"); @out.WriteLine(" <queue name=\"childA2\" />"); @out.WriteLine(" <queue name=\"childA3\" />"); @out.WriteLine(" <queue name=\"childA4\" />"); @out.WriteLine(" </queue>"); @out.WriteLine(" <queue name=\"parentB\" >"); @out.WriteLine(" <weight>1</weight>"); @out.WriteLine(" <queue name=\"childB1\" />"); @out.WriteLine(" <queue name=\"childB2\" />"); @out.WriteLine(" </queue>"); @out.WriteLine("</queue>"); @out.WriteLine("<defaultQueueSchedulingPolicy>" + policy + "</defaultQueueSchedulingPolicy>" ); @out.WriteLine("</allocations>"); @out.Close(); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(mem, vCores), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.Handle(nodeEvent1); }
// Test RM restarts after AM container is preempted, new RM should not count // AM preemption failure towards the max-retry-account and should be able to // re-launch the AM. /// <exception cref="System.Exception"/> public virtual void TestPreemptedAMRestartOnRMRestart() { YarnConfiguration conf = new YarnConfiguration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler )); conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true); conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false); conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName); // explicitly set max-am-retry count as 1. conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); MockRM rm1 = new MockRM(conf, memStore); rm1.Start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService()); nm1.RegisterNode(); RMApp app1 = rm1.SubmitApp(200); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler(); ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId( ), 1); // Forcibly preempt the am container; scheduler.KillContainer(scheduler.GetRMContainer(amContainer)); am1.WaitForState(RMAppAttemptState.Failed); NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry()); rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted); // state store has 1 attempt stored. ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId ()]; NUnit.Framework.Assert.AreEqual(1, appState.GetAttemptCount()); // attempt stored has the preempted container exit status. NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Preempted, appState.GetAttempt (am1.GetApplicationAttemptId()).GetAMContainerExitStatus()); // Restart rm. MockRM rm2 = new MockRM(conf, memStore); nm1.SetResourceTrackerService(rm2.GetResourceTrackerService()); nm1.RegisterNode(); rm2.Start(); // Restarted RM should re-launch the am. MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1); MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2); RMAppAttempt attempt2 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt (); NUnit.Framework.Assert.IsTrue(attempt2.ShouldCountTowardsMaxAttemptRetry()); NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt( am2.GetApplicationAttemptId()).GetAMContainerExitStatus()); rm1.Stop(); rm2.Stop(); }
public virtual void Setup() { Configuration conf = CreateConfiguration(); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); }
private void CheckUsedResource(MockRM rm, string queueName, int memory, string label ) { CapacityScheduler scheduler = (CapacityScheduler)rm.GetResourceScheduler(); CSQueue queue = scheduler.GetQueue(queueName); NUnit.Framework.Assert.AreEqual(memory, queue.GetQueueResourceUsage().GetUsed(label ).GetMemory()); }
public virtual void TestConcurrentAccess() { conf.Set(FairSchedulerConfiguration.AssignMultiple, "false"); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); string queueName = "root.queue1"; FSLeafQueue schedulable = scheduler.GetQueueManager().GetLeafQueue(queueName, true ); ApplicationAttemptId applicationAttemptId = CreateAppAttemptId(1, 1); RMContext rmContext = resourceManager.GetRMContext(); FSAppAttempt app = new FSAppAttempt(scheduler, applicationAttemptId, "user1", schedulable , null, rmContext); // this needs to be in sync with the number of runnables declared below int testThreads = 2; IList <Runnable> runnables = new AList <Runnable>(); // add applications to modify the list runnables.AddItem(new _Runnable_257(schedulable, app)); // iterate over the list a couple of times in a different thread runnables.AddItem(new _Runnable_267(schedulable)); IList <Exception> exceptions = Sharpen.Collections.SynchronizedList(new AList <Exception >()); ExecutorService threadPool = Executors.NewFixedThreadPool(testThreads); try { CountDownLatch allExecutorThreadsReady = new CountDownLatch(testThreads); CountDownLatch startBlocker = new CountDownLatch(1); CountDownLatch allDone = new CountDownLatch(testThreads); foreach (Runnable submittedTestRunnable in runnables) { threadPool.Submit(new _Runnable_287(allExecutorThreadsReady, startBlocker, submittedTestRunnable , exceptions, allDone)); } // wait until all threads are ready allExecutorThreadsReady.Await(); // start all test runners startBlocker.CountDown(); int testTimeout = 2; NUnit.Framework.Assert.IsTrue("Timeout waiting for more than " + testTimeout + " seconds" , allDone.Await(testTimeout, TimeUnit.Seconds)); } catch (Exception ie) { exceptions.AddItem(ie); } finally { threadPool.ShutdownNow(); } NUnit.Framework.Assert.IsTrue("Test failed with exception(s)" + exceptions, exceptions .IsEmpty()); }
public virtual void TestMaxAllocationAfterUpdateNodeResource() { int configuredMaxVCores = 20; int configuredMaxMemory = 10 * 1024; Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(configuredMaxMemory, configuredMaxVCores); ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores ); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); MockRM rm = new MockRM(conf); try { rm.Start(); AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler( ); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2048, 5); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(4096, 10); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource3 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(512, 1); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource4 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1024, 2); RMNode node1 = MockNodes.NewNodeInfo(0, resource1, 1, "127.0.0.2"); scheduler.Handle(new NodeAddedSchedulerEvent(node1)); RMNode node2 = MockNodes.NewNodeInfo(0, resource3, 2, "127.0.0.3"); scheduler.Handle(new NodeAddedSchedulerEvent(node2)); VerifyMaximumResourceCapability(resource1, scheduler); // increase node1 resource scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource2, 0)); VerifyMaximumResourceCapability(resource2, scheduler); // decrease node1 resource scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource1, 0)); VerifyMaximumResourceCapability(resource1, scheduler); // increase node2 resource scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource4, 0)); VerifyMaximumResourceCapability(resource1, scheduler); // decrease node2 resource scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource3, 0)); VerifyMaximumResourceCapability(resource1, scheduler); } finally { rm.Stop(); } }
public virtual void TestMetricsInitializedOnRMInit() { YarnConfiguration conf = new YarnConfiguration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics(); CheckApps(metrics, 0, 0, 0, 0, 0, 0, true); MetricsAsserts.AssertGauge("ReservedContainers", 0, metrics); }
public virtual void Setup() { mockClock = new FairSchedulerTestBase.MockClock(); conf = CreateConfiguration(); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); scheduler.SetClock(mockClock); NUnit.Framework.Assert.IsTrue(scheduler.IsContinuousSchedulingEnabled()); NUnit.Framework.Assert.AreEqual(FairSchedulerConfiguration.DefaultContinuousSchedulingSleepMs , scheduler.GetContinuousSchedulingSleepMs()); NUnit.Framework.Assert.AreEqual(mockClock, scheduler.GetClock()); }
public virtual void TestRefreshQueuesWithReservations() { CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler(); // Test add one reservation dynamically and manually modify capacity ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a")); cs.AddQueue(a1); a1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f)); // Test add another reservation queue and use setEntitlement to modify // capacity ReservationQueue a2 = new ReservationQueue(cs, "a2", (PlanQueue)cs.GetQueue("a")); cs.AddQueue(a2); cs.SetEntitlement("a2", new QueueEntitlement(A2Capacity / 100, 1.0f)); // Verify all allocations match tcs.CheckQueueCapacities(cs, ACapacity, BCapacity); // Reinitialize and verify all dynamic queued survived CapacitySchedulerConfiguration conf = cs.GetConfiguration(); conf.SetCapacity(A, 80f); conf.SetCapacity(B, 20f); cs.Reinitialize(conf, rm.GetRMContext()); tcs.CheckQueueCapacities(cs, 80f, 20f); }
/// <exception cref="System.Exception"/> public virtual void Test() { conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile); PrintWriter @out = new PrintWriter(new FileWriter(AllocFile)); @out.WriteLine("<?xml version=\"1.0\"?>"); @out.WriteLine("<allocations>"); @out.WriteLine("<queue name=\"queueA\">"); @out.WriteLine("<minResources>2048mb,0vcores</minResources>"); @out.WriteLine("</queue>"); @out.WriteLine("<queue name=\"queueB\">"); @out.WriteLine("<minResources>2048mb,0vcores</minResources>"); @out.WriteLine("</queue>"); @out.WriteLine("</allocations>"); @out.Close(); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(4 * 1024, 4), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.Handle(nodeEvent1); scheduler.Update(); // Queue A wants 3 * 1024. Node update gives this all to A CreateSchedulingRequest(3 * 1024, "queueA", "user1"); scheduler.Update(); NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1); scheduler.Handle(nodeEvent2); // Queue B arrives and wants 1 * 1024 CreateSchedulingRequest(1 * 1024, "queueB", "user1"); scheduler.Update(); ICollection <FSLeafQueue> queues = scheduler.GetQueueManager().GetLeafQueues(); NUnit.Framework.Assert.AreEqual(3, queues.Count); // Queue A should be above min share, B below. FSLeafQueue queueA = scheduler.GetQueueManager().GetLeafQueue("queueA", false); FSLeafQueue queueB = scheduler.GetQueueManager().GetLeafQueue("queueB", false); NUnit.Framework.Assert.IsFalse(queueA.IsStarvedForMinShare()); NUnit.Framework.Assert.IsTrue(queueB.IsStarvedForMinShare()); // Node checks in again, should allocate for B scheduler.Handle(nodeEvent2); // Now B should have min share ( = demand here) NUnit.Framework.Assert.IsFalse(queueB.IsStarvedForMinShare()); }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyClusterMetrics(int submittedApps, int completedApps, int reservedMB, int availableMB, int allocMB, int reservedVirtualCores, int availableVirtualCores , int allocVirtualCores, int totalVirtualCores, int containersAlloc, int totalMB , int totalNodes, int lostNodes, int unhealthyNodes, int decommissionedNodes, int rebootedNodes, int activeNodes) { ResourceScheduler rs = rm.GetResourceScheduler(); QueueMetrics metrics = rs.GetRootQueueMetrics(); ClusterMetrics clusterMetrics = ClusterMetrics.GetMetrics(); long totalMBExpect = metrics.GetAvailableMB() + metrics.GetAllocatedMB(); long totalVirtualCoresExpect = metrics.GetAvailableVirtualCores() + metrics.GetAllocatedVirtualCores (); NUnit.Framework.Assert.AreEqual("appsSubmitted doesn't match", metrics.GetAppsSubmitted (), submittedApps); NUnit.Framework.Assert.AreEqual("appsCompleted doesn't match", metrics.GetAppsCompleted (), completedApps); NUnit.Framework.Assert.AreEqual("reservedMB doesn't match", metrics.GetReservedMB (), reservedMB); NUnit.Framework.Assert.AreEqual("availableMB doesn't match", metrics.GetAvailableMB (), availableMB); NUnit.Framework.Assert.AreEqual("allocatedMB doesn't match", metrics.GetAllocatedMB (), allocMB); NUnit.Framework.Assert.AreEqual("reservedVirtualCores doesn't match", metrics.GetReservedVirtualCores (), reservedVirtualCores); NUnit.Framework.Assert.AreEqual("availableVirtualCores doesn't match", metrics.GetAvailableVirtualCores (), availableVirtualCores); NUnit.Framework.Assert.AreEqual("allocatedVirtualCores doesn't match", totalVirtualCoresExpect , allocVirtualCores); NUnit.Framework.Assert.AreEqual("containersAllocated doesn't match", 0, containersAlloc ); NUnit.Framework.Assert.AreEqual("totalMB doesn't match", totalMBExpect, totalMB); NUnit.Framework.Assert.AreEqual("totalNodes doesn't match", clusterMetrics.GetNumActiveNMs () + clusterMetrics.GetNumLostNMs() + clusterMetrics.GetNumDecommisionedNMs() + clusterMetrics.GetNumRebootedNMs() + clusterMetrics.GetUnhealthyNMs(), totalNodes ); NUnit.Framework.Assert.AreEqual("lostNodes doesn't match", clusterMetrics.GetNumLostNMs (), lostNodes); NUnit.Framework.Assert.AreEqual("unhealthyNodes doesn't match", clusterMetrics.GetUnhealthyNMs (), unhealthyNodes); NUnit.Framework.Assert.AreEqual("decommissionedNodes doesn't match", clusterMetrics .GetNumDecommisionedNMs(), decommissionedNodes); NUnit.Framework.Assert.AreEqual("rebootedNodes doesn't match", clusterMetrics.GetNumRebootedNMs (), rebootedNodes); NUnit.Framework.Assert.AreEqual("activeNodes doesn't match", clusterMetrics.GetNumActiveNMs (), activeNodes); }
public virtual void TestMaximimumAllocationMemory() { int node1MaxMemory = 15 * 1024; int node2MaxMemory = 5 * 1024; int node3MaxMemory = 6 * 1024; int configuredMaxMemory = 10 * 1024; ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 1000 * 1000 ); MockRM rm = new MockRM(conf); try { rm.Start(); TestMaximumAllocationMemoryHelper((AbstractYarnScheduler)rm.GetResourceScheduler( ), node1MaxMemory, node2MaxMemory, node3MaxMemory, configuredMaxMemory, configuredMaxMemory , configuredMaxMemory, configuredMaxMemory, configuredMaxMemory, configuredMaxMemory ); } finally { rm.Stop(); } conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); rm = new MockRM(conf); try { rm.Start(); TestMaximumAllocationMemoryHelper((AbstractYarnScheduler)rm.GetResourceScheduler( ), node1MaxMemory, node2MaxMemory, node3MaxMemory, configuredMaxMemory, configuredMaxMemory , configuredMaxMemory, node2MaxMemory, node3MaxMemory, node2MaxMemory); } finally { rm.Stop(); } }
public virtual void TestMaximimumAllocationVCores() { int node1MaxVCores = 15; int node2MaxVCores = 5; int node3MaxVCores = 6; int configuredMaxVCores = 10; ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 1000 * 1000 ); MockRM rm = new MockRM(conf); try { rm.Start(); TestMaximumAllocationVCoresHelper((AbstractYarnScheduler)rm.GetResourceScheduler( ), node1MaxVCores, node2MaxVCores, node3MaxVCores, configuredMaxVCores, configuredMaxVCores , configuredMaxVCores, configuredMaxVCores, configuredMaxVCores, configuredMaxVCores ); } finally { rm.Stop(); } conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); rm = new MockRM(conf); try { rm.Start(); TestMaximumAllocationVCoresHelper((AbstractYarnScheduler)rm.GetResourceScheduler( ), node1MaxVCores, node2MaxVCores, node3MaxVCores, configuredMaxVCores, configuredMaxVCores , configuredMaxVCores, node2MaxVCores, node3MaxVCores, node2MaxVCores); } finally { rm.Stop(); } }
/// <exception cref="Org.Codehaus.Jettison.Json.JSONException"/> /// <exception cref="System.Exception"/> public virtual void VerifyNodeInfoGeneric(MockNM nm, string state, string rack, string id, string nodeHostName, string nodeHTTPAddress, long lastHealthUpdate, string healthReport, int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores , long availVirtualCores, string version) { RMNode node = rm.GetRMContext().GetRMNodes()[nm.GetNodeId()]; ResourceScheduler sched = rm.GetResourceScheduler(); SchedulerNodeReport report = sched.GetNodeReport(nm.GetNodeId()); WebServicesTestUtils.CheckStringMatch("state", node.GetState().ToString(), state); WebServicesTestUtils.CheckStringMatch("rack", node.GetRackName(), rack); WebServicesTestUtils.CheckStringMatch("id", nm.GetNodeId().ToString(), id); WebServicesTestUtils.CheckStringMatch("nodeHostName", nm.GetNodeId().GetHost(), nodeHostName ); WebServicesTestUtils.CheckStringMatch("healthReport", node.GetHealthReport().ToString (), healthReport); string expectedHttpAddress = nm.GetNodeId().GetHost() + ":" + nm.GetHttpPort(); WebServicesTestUtils.CheckStringMatch("nodeHTTPAddress", expectedHttpAddress, nodeHTTPAddress ); WebServicesTestUtils.CheckStringMatch("version", node.GetNodeManagerVersion(), version ); long expectedHealthUpdate = node.GetLastHealthReportTime(); NUnit.Framework.Assert.AreEqual("lastHealthUpdate doesn't match, got: " + lastHealthUpdate + " expected: " + expectedHealthUpdate, expectedHealthUpdate, lastHealthUpdate); if (report != null) { NUnit.Framework.Assert.AreEqual("numContainers doesn't match: " + numContainers, report.GetNumContainers(), numContainers); NUnit.Framework.Assert.AreEqual("usedMemoryMB doesn't match: " + usedMemoryMB, report .GetUsedResource().GetMemory(), usedMemoryMB); NUnit.Framework.Assert.AreEqual("availMemoryMB doesn't match: " + availMemoryMB, report.GetAvailableResource().GetMemory(), availMemoryMB); NUnit.Framework.Assert.AreEqual("usedVirtualCores doesn't match: " + usedVirtualCores , report.GetUsedResource().GetVirtualCores(), usedVirtualCores); NUnit.Framework.Assert.AreEqual("availVirtualCores doesn't match: " + availVirtualCores , report.GetAvailableResource().GetVirtualCores(), availVirtualCores); } }
/// <exception cref="System.Exception"/> private void CheckAppQueue(MockRM resourceManager, string user, string submissionQueue , string expected) { RMApp app = resourceManager.SubmitApp(200, "name", user, new Dictionary <ApplicationAccessType , string>(), false, submissionQueue, -1, null, "MAPREDUCE", false); RMAppState expectedState = expected.IsEmpty() ? RMAppState.Failed : RMAppState.Accepted; resourceManager.WaitForState(app.GetApplicationId(), expectedState); // get scheduler app CapacityScheduler cs = (CapacityScheduler)resourceManager.GetResourceScheduler(); SchedulerApplication schedulerApp = cs.GetSchedulerApplications()[app.GetApplicationId ()]; string queue = string.Empty; if (schedulerApp != null) { queue = schedulerApp.GetQueue().GetQueueName(); } NUnit.Framework.Assert.IsTrue("expected " + expected + " actual " + queue, expected .Equals(queue)); NUnit.Framework.Assert.AreEqual(expected, app.GetQueue()); }
public virtual void TestUpdateDemand() { conf.Set(FairSchedulerConfiguration.AssignMultiple, "false"); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); scheduler.allocConf = Org.Mockito.Mockito.Mock <AllocationConfiguration>(); string queueName = "root.queue1"; Org.Mockito.Mockito.When(scheduler.allocConf.GetMaxResources(queueName)).ThenReturn (maxResource); Org.Mockito.Mockito.When(scheduler.allocConf.GetMinResources(queueName)).ThenReturn (Resources.None()); FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null); FSAppAttempt app = Org.Mockito.Mockito.Mock <FSAppAttempt>(); Org.Mockito.Mockito.When(app.GetDemand()).ThenReturn(maxResource); schedulable.AddAppSchedulable(app); schedulable.AddAppSchedulable(app); schedulable.UpdateDemand(); NUnit.Framework.Assert.IsTrue("Demand is greater than max allowed ", Resources.Equals (schedulable.GetDemand(), maxResource)); }
public virtual void TestBlackListNodes() { Configuration conf = new Configuration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); rm.Start(); FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler(); string host = "127.0.0.1"; RMNode node = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, host); fs.Handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = BuilderUtils.NewApplicationId(100, 1); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); CreateMockRMApp(appAttemptId, rm.GetRMContext()); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "default", "user"); fs.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false ); fs.Handle(attemptEvent); // Verify the blacklist can be updated independent of requesting containers fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections .EmptyList <ContainerId>(), Sharpen.Collections.SingletonList(host), null); NUnit.Framework.Assert.IsTrue(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted (host)); fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections .EmptyList <ContainerId>(), null, Sharpen.Collections.SingletonList(host)); NUnit.Framework.Assert.IsFalse(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted (host)); rm.Stop(); }
// AM container preempted, nm disk failure // should not be counted towards AM max retry count. /// <exception cref="System.Exception"/> public virtual void TestShouldNotCountFailureToMaxAttemptRetry() { YarnConfiguration conf = new YarnConfiguration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler )); // explicitly set max-am-retry count as 1. conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1); conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true); conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName); MemoryRMStateStore memStore = new MemoryRMStateStore(); memStore.Init(conf); MockRM rm1 = new MockRM(conf, memStore); rm1.Start(); MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService()); nm1.RegisterNode(); RMApp app1 = rm1.SubmitApp(200); RMAppAttempt attempt1 = app1.GetCurrentAppAttempt(); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler(); ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId( ), 1); // Preempt the first attempt; scheduler.KillContainer(scheduler.GetRMContainer(amContainer)); am1.WaitForState(RMAppAttemptState.Failed); NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry()); rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted); ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId ()]; // AM should be restarted even though max-am-attempt is 1. MockAM am2 = rm1.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1); RMAppAttempt attempt2 = app1.GetCurrentAppAttempt(); NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt2).MayBeLastAttempt()); // Preempt the second attempt. ContainerId amContainer2 = ContainerId.NewContainerId(am2.GetApplicationAttemptId (), 1); scheduler.KillContainer(scheduler.GetRMContainer(amContainer2)); am2.WaitForState(RMAppAttemptState.Failed); NUnit.Framework.Assert.IsTrue(!attempt2.ShouldCountTowardsMaxAttemptRetry()); rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted); MockAM am3 = rm1.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 3, nm1); RMAppAttempt attempt3 = app1.GetCurrentAppAttempt(); NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt3).MayBeLastAttempt()); // mimic NM disk_failure ContainerStatus containerStatus = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerStatus >(); containerStatus.SetContainerId(attempt3.GetMasterContainer().GetId()); containerStatus.SetDiagnostics("mimic NM disk_failure"); containerStatus.SetState(ContainerState.Complete); containerStatus.SetExitStatus(ContainerExitStatus.DisksFailed); IDictionary <ApplicationId, IList <ContainerStatus> > conts = new Dictionary <ApplicationId , IList <ContainerStatus> >(); conts[app1.GetApplicationId()] = Sharpen.Collections.SingletonList(containerStatus ); nm1.NodeHeartbeat(conts, true); am3.WaitForState(RMAppAttemptState.Failed); NUnit.Framework.Assert.IsTrue(!attempt3.ShouldCountTowardsMaxAttemptRetry()); NUnit.Framework.Assert.AreEqual(ContainerExitStatus.DisksFailed, appState.GetAttempt (am3.GetApplicationAttemptId()).GetAMContainerExitStatus()); rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted); MockAM am4 = rm1.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 4, nm1); RMAppAttempt attempt4 = app1.GetCurrentAppAttempt(); NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt4).MayBeLastAttempt()); // create second NM, and register to rm1 MockNM nm2 = new MockNM("127.0.0.1:2234", 8000, rm1.GetResourceTrackerService()); nm2.RegisterNode(); // nm1 heartbeats to report unhealthy // This will mimic ContainerExitStatus.ABORT nm1.NodeHeartbeat(false); am4.WaitForState(RMAppAttemptState.Failed); NUnit.Framework.Assert.IsTrue(!attempt4.ShouldCountTowardsMaxAttemptRetry()); NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, appState.GetAttempt( am4.GetApplicationAttemptId()).GetAMContainerExitStatus()); // launch next AM in nm2 nm2.NodeHeartbeat(true); MockAM am5 = rm1.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 5, nm2); RMAppAttempt attempt5 = app1.GetCurrentAppAttempt(); NUnit.Framework.Assert.IsTrue(((RMAppAttemptImpl)attempt5).MayBeLastAttempt()); // fail the AM normally nm2.NodeHeartbeat(am5.GetApplicationAttemptId(), 1, ContainerState.Complete); am5.WaitForState(RMAppAttemptState.Failed); NUnit.Framework.Assert.IsTrue(attempt5.ShouldCountTowardsMaxAttemptRetry()); // AM should not be restarted. rm1.WaitForState(app1.GetApplicationId(), RMAppState.Failed); NUnit.Framework.Assert.AreEqual(5, app1.GetAppAttempts().Count); rm1.Stop(); }
/// <exception cref="System.Exception"/> public virtual void TestAMRestartWithExistingContainers() { YarnConfiguration conf = new YarnConfiguration(); conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 2); MockRM rm1 = new MockRM(conf); rm1.Start(); RMApp app1 = rm1.SubmitApp(200, "name", "user", new Dictionary <ApplicationAccessType , string>(), false, "default", -1, null, "MAPREDUCE", false, true); MockNM nm1 = new MockNM("127.0.0.1:1234", 10240, rm1.GetResourceTrackerService()); nm1.RegisterNode(); MockNM nm2 = new MockNM("127.0.0.1:2351", 4089, rm1.GetResourceTrackerService()); nm2.RegisterNode(); MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1); int NumContainers = 3; // allocate NUM_CONTAINERS containers am1.Allocate("127.0.0.1", 1024, NumContainers, new AList <ContainerId>()); nm1.NodeHeartbeat(true); // wait for containers to be allocated. IList <Container> containers = am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers(); while (containers.Count != NumContainers) { nm1.NodeHeartbeat(true); Sharpen.Collections.AddAll(containers, am1.Allocate(new AList <ResourceRequest>(), new AList <ContainerId>()).GetAllocatedContainers()); Sharpen.Thread.Sleep(200); } // launch the 2nd container, for testing running container transferred. nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 2, ContainerState.Running); ContainerId containerId2 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 2); rm1.WaitForState(nm1, containerId2, RMContainerState.Running); // launch the 3rd container, for testing container allocated by previous // attempt is completed by the next new attempt/ nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 3, ContainerState.Running); ContainerId containerId3 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 3); rm1.WaitForState(nm1, containerId3, RMContainerState.Running); // 4th container still in AQUIRED state. for testing Acquired container is // always killed. ContainerId containerId4 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 4); rm1.WaitForState(nm1, containerId4, RMContainerState.Acquired); // 5th container is in Allocated state. for testing allocated container is // always killed. am1.Allocate("127.0.0.1", 1024, 1, new AList <ContainerId>()); nm1.NodeHeartbeat(true); ContainerId containerId5 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 5); rm1.WaitForContainerAllocated(nm1, containerId5); rm1.WaitForState(nm1, containerId5, RMContainerState.Allocated); // 6th container is in Reserved state. am1.Allocate("127.0.0.1", 6000, 1, new AList <ContainerId>()); ContainerId containerId6 = ContainerId.NewContainerId(am1.GetApplicationAttemptId (), 6); nm1.NodeHeartbeat(true); SchedulerApplicationAttempt schedulerAttempt = ((AbstractYarnScheduler)rm1.GetResourceScheduler ()).GetCurrentAttemptForContainer(containerId6); while (schedulerAttempt.GetReservedContainers().IsEmpty()) { System.Console.Out.WriteLine("Waiting for container " + containerId6 + " to be reserved." ); nm1.NodeHeartbeat(true); Sharpen.Thread.Sleep(200); } // assert containerId6 is reserved. NUnit.Framework.Assert.AreEqual(containerId6, schedulerAttempt.GetReservedContainers ()[0].GetContainerId()); // fail the AM by sending CONTAINER_FINISHED event without registering. nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 1, ContainerState.Complete); am1.WaitForState(RMAppAttemptState.Failed); // wait for some time. previous AM's running containers should still remain // in scheduler even though am failed Sharpen.Thread.Sleep(3000); rm1.WaitForState(nm1, containerId2, RMContainerState.Running); // acquired/allocated containers are cleaned up. NUnit.Framework.Assert.IsNull(rm1.GetResourceScheduler().GetRMContainer(containerId4 )); NUnit.Framework.Assert.IsNull(rm1.GetResourceScheduler().GetRMContainer(containerId5 )); // wait for app to start a new attempt. rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted); // assert this is a new AM. ApplicationAttemptId newAttemptId = app1.GetCurrentAppAttempt().GetAppAttemptId(); NUnit.Framework.Assert.IsFalse(newAttemptId.Equals(am1.GetApplicationAttemptId()) ); // launch the new AM RMAppAttempt attempt2 = app1.GetCurrentAppAttempt(); nm1.NodeHeartbeat(true); MockAM am2 = rm1.SendAMLaunched(attempt2.GetAppAttemptId()); RegisterApplicationMasterResponse registerResponse = am2.RegisterAppAttempt(); // Assert two containers are running: container2 and container3; NUnit.Framework.Assert.AreEqual(2, registerResponse.GetContainersFromPreviousAttempts ().Count); bool containerId2Exists = false; bool containerId3Exists = false; foreach (Container container in registerResponse.GetContainersFromPreviousAttempts ()) { if (container.GetId().Equals(containerId2)) { containerId2Exists = true; } if (container.GetId().Equals(containerId3)) { containerId3Exists = true; } } NUnit.Framework.Assert.IsTrue(containerId2Exists && containerId3Exists); rm1.WaitForState(app1.GetApplicationId(), RMAppState.Running); // complete container by sending the container complete event which has earlier // attempt's attemptId nm1.NodeHeartbeat(am1.GetApplicationAttemptId(), 3, ContainerState.Complete); // Even though the completed container containerId3 event was sent to the // earlier failed attempt, new RMAppAttempt can also capture this container // info. // completed containerId4 is also transferred to the new attempt. RMAppAttempt newAttempt = app1.GetRMAppAttempt(am2.GetApplicationAttemptId()); // 4 containers finished, acquired/allocated/reserved/completed. WaitForContainersToFinish(4, newAttempt); bool container3Exists = false; bool container4Exists = false; bool container5Exists = false; bool container6Exists = false; foreach (ContainerStatus status in newAttempt.GetJustFinishedContainers()) { if (status.GetContainerId().Equals(containerId3)) { // containerId3 is the container ran by previous attempt but finished by the // new attempt. container3Exists = true; } if (status.GetContainerId().Equals(containerId4)) { // containerId4 is the Acquired Container killed by the previous attempt, // it's now inside new attempt's finished container list. container4Exists = true; } if (status.GetContainerId().Equals(containerId5)) { // containerId5 is the Allocated container killed by previous failed attempt. container5Exists = true; } if (status.GetContainerId().Equals(containerId6)) { // containerId6 is the reserved container killed by previous failed attempt. container6Exists = true; } } NUnit.Framework.Assert.IsTrue(container3Exists && container4Exists && container5Exists && container6Exists); // New SchedulerApplicationAttempt also has the containers info. rm1.WaitForState(nm1, containerId2, RMContainerState.Running); // record the scheduler attempt for testing. SchedulerApplicationAttempt schedulerNewAttempt = ((AbstractYarnScheduler)rm1.GetResourceScheduler ()).GetCurrentAttemptForContainer(containerId2); // finish this application MockRM.FinishAMAndVerifyAppState(app1, rm1, nm1, am2); // the 2nd attempt released the 1st attempt's running container, when the // 2nd attempt finishes. NUnit.Framework.Assert.IsFalse(schedulerNewAttempt.GetLiveContainers().Contains(containerId2 )); // all 4 normal containers finished. System.Console.Out.WriteLine("New attempt's just finished containers: " + newAttempt .GetJustFinishedContainers()); WaitForContainersToFinish(5, newAttempt); rm1.Stop(); }
/// <exception cref="System.Exception"/> public virtual void TestIsStarvedForFairShare() { conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile); PrintWriter @out = new PrintWriter(new FileWriter(AllocFile)); @out.WriteLine("<?xml version=\"1.0\"?>"); @out.WriteLine("<allocations>"); @out.WriteLine("<queue name=\"queueA\">"); @out.WriteLine("<weight>.2</weight>"); @out.WriteLine("</queue>"); @out.WriteLine("<queue name=\"queueB\">"); @out.WriteLine("<weight>.8</weight>"); @out.WriteLine("<fairSharePreemptionThreshold>.4</fairSharePreemptionThreshold>"); @out.WriteLine("<queue name=\"queueB1\">"); @out.WriteLine("</queue>"); @out.WriteLine("<queue name=\"queueB2\">"); @out.WriteLine("<fairSharePreemptionThreshold>.6</fairSharePreemptionThreshold>"); @out.WriteLine("</queue>"); @out.WriteLine("</queue>"); @out.WriteLine("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>" ); @out.WriteLine("</allocations>"); @out.Close(); resourceManager = new MockRM(conf); resourceManager.Start(); scheduler = (FairScheduler)resourceManager.GetResourceScheduler(); // Add one big node (only care about aggregate capacity) RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(10 * 1024, 10), 1, "127.0.0.1"); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1); scheduler.Handle(nodeEvent1); scheduler.Update(); // Queue A wants 4 * 1024. Node update gives this all to A CreateSchedulingRequest(1 * 1024, "queueA", "user1", 4); scheduler.Update(); NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1); for (int i = 0; i < 4; i++) { scheduler.Handle(nodeEvent2); } QueueManager queueMgr = scheduler.GetQueueManager(); FSLeafQueue queueA = queueMgr.GetLeafQueue("queueA", false); NUnit.Framework.Assert.AreEqual(4 * 1024, queueA.GetResourceUsage().GetMemory()); // Both queue B1 and queue B2 want 3 * 1024 CreateSchedulingRequest(1 * 1024, "queueB.queueB1", "user1", 3); CreateSchedulingRequest(1 * 1024, "queueB.queueB2", "user1", 3); scheduler.Update(); for (int i_1 = 0; i_1 < 4; i_1++) { scheduler.Handle(nodeEvent2); } FSLeafQueue queueB1 = queueMgr.GetLeafQueue("queueB.queueB1", false); FSLeafQueue queueB2 = queueMgr.GetLeafQueue("queueB.queueB2", false); NUnit.Framework.Assert.AreEqual(2 * 1024, queueB1.GetResourceUsage().GetMemory()); NUnit.Framework.Assert.AreEqual(2 * 1024, queueB2.GetResourceUsage().GetMemory()); // For queue B1, the fairSharePreemptionThreshold is 0.4, and the fair share // threshold is 1.6 * 1024 NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare()); // For queue B2, the fairSharePreemptionThreshold is 0.6, and the fair share // threshold is 2.4 * 1024 NUnit.Framework.Assert.IsTrue(queueB2.IsStarvedForFairShare()); // Node checks in again scheduler.Handle(nodeEvent2); scheduler.Handle(nodeEvent2); NUnit.Framework.Assert.AreEqual(3 * 1024, queueB1.GetResourceUsage().GetMemory()); NUnit.Framework.Assert.AreEqual(3 * 1024, queueB2.GetResourceUsage().GetMemory()); // Both queue B1 and queue B2 usages go to 3 * 1024 NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare()); NUnit.Framework.Assert.IsFalse(queueB2.IsStarvedForFairShare()); }
/// <exception cref="System.Exception"/> public virtual void TestQueueMapping() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); SetupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(csConf); CapacityScheduler cs = new CapacityScheduler(); RMContext rmContext = TestUtils.GetMockRMContext(); cs.SetConf(conf); cs.SetRMContext(rmContext); cs.Init(conf); cs.Start(); conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(SimpleGroupsMapping ), typeof(GroupMappingServiceProvider)); conf.Set(CapacitySchedulerConfiguration.EnableQueueMappingOverride, "true"); // configuration parsing tests - negative test cases CheckInvalidQMapping(conf, cs, "x:a:b", "invalid specifier"); CheckInvalidQMapping(conf, cs, "u:a", "no queue specified"); CheckInvalidQMapping(conf, cs, "g:a", "no queue specified"); CheckInvalidQMapping(conf, cs, "u:a:b,g:a", "multiple mappings with invalid mapping" ); CheckInvalidQMapping(conf, cs, "u:a:b,g:a:d:e", "too many path segments"); CheckInvalidQMapping(conf, cs, "u::", "empty source and queue"); CheckInvalidQMapping(conf, cs, "u:", "missing source missing queue"); CheckInvalidQMapping(conf, cs, "u:a:", "empty source missing q"); // simple base case for mapping user to queue conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:a:" + Q1); cs.Reinitialize(conf, null); CheckQMapping("a", Q1, cs); // group mapping test conf.Set(CapacitySchedulerConfiguration.QueueMapping, "g:agroup:" + Q1); cs.Reinitialize(conf, null); CheckQMapping("a", Q1, cs); // %user tests conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:%user:"******"a", Q2, cs); conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:%user:%user"); cs.Reinitialize(conf, null); CheckQMapping("a", "a", cs); // %primary_group tests conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:%user:%primary_group"); cs.Reinitialize(conf, null); CheckQMapping("a", "agroup", cs); // non-primary group mapping conf.Set(CapacitySchedulerConfiguration.QueueMapping, "g:asubgroup1:" + Q1); cs.Reinitialize(conf, null); CheckQMapping("a", Q1, cs); // space trimming conf.Set(CapacitySchedulerConfiguration.QueueMapping, " u : a : " + Q1); cs.Reinitialize(conf, null); CheckQMapping("a", Q1, cs); csConf = new CapacitySchedulerConfiguration(); csConf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).FullName); SetupQueueConfiguration(csConf); conf = new YarnConfiguration(csConf); resourceManager = new MockRM(csConf); resourceManager.Start(); conf.SetClass(CommonConfigurationKeys.HadoopSecurityGroupMapping, typeof(SimpleGroupsMapping ), typeof(GroupMappingServiceProvider)); conf.Set(CapacitySchedulerConfiguration.EnableQueueMappingOverride, "true"); conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:user:"******"user", Q2, Q1); // toggle admin override and retry conf.SetBoolean(CapacitySchedulerConfiguration.EnableQueueMappingOverride, false); conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:user:"******"user", Q2, Q2); // ensure that if a user does not specify a Q, the user mapping is used CheckAppQueue(resourceManager, "user", null, Q1); conf.Set(CapacitySchedulerConfiguration.QueueMapping, "g:usergroup:" + Q2); SetupQueueConfiguration(csConf); resourceManager.GetResourceScheduler().Reinitialize(conf, null); // ensure that if a user does not specify a Q, the group mapping is used CheckAppQueue(resourceManager, "user", null, Q2); // if the mapping specifies a queue that does not exist, the job is rejected conf.Set(CapacitySchedulerConfiguration.QueueMapping, "u:user:non_existent_queue" ); SetupQueueConfiguration(csConf); bool fail = false; try { resourceManager.GetResourceScheduler().Reinitialize(conf, null); } catch (IOException) { fail = true; } NUnit.Framework.Assert.IsTrue("queue initialization failed for non-existent q", fail ); resourceManager.Stop(); }
public virtual void TestAddAndRemoveAppFromFiFoScheduler() { Configuration conf = new Configuration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); AbstractYarnScheduler <SchedulerApplicationAttempt, SchedulerNode> fs = (AbstractYarnScheduler <SchedulerApplicationAttempt, SchedulerNode>)rm.GetResourceScheduler(); TestSchedulerUtils.VerifyAppAddedAndRemovedFromScheduler(fs.GetSchedulerApplications (), fs, "queue"); }