public virtual void TestFairShareResetsToZeroWhenAppsComplete() { int nodeCapacity = 16 * 1024; CreateClusterWithQueuesAndOneNode(nodeCapacity, "fair"); // Run apps in childA1,childA2 which are under parentA ApplicationAttemptId app1 = CreateSchedulingRequest(2 * 1024, "root.parentA.childA1" , "user1"); ApplicationAttemptId app2 = CreateSchedulingRequest(3 * 1024, "root.parentA.childA2" , "user2"); scheduler.Update(); // Verify if both the active queues under parentA get 50% fair // share for (int i = 1; i <= 2; i++) { NUnit.Framework.Assert.AreEqual(50, (double)scheduler.GetQueueManager().GetLeafQueue ("root.parentA.childA" + i, false).GetFairShare().GetMemory() / nodeCapacity * 100 , .9); } // Let app under childA1 complete. This should cause the fair share // of queue childA1 to be reset to zero,since the queue has no apps running. // Queue childA2's fair share would increase to 100% since its the only // active queue. AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent (app1, RMAppAttemptState.Finished, false); scheduler.Handle(appRemovedEvent1); scheduler.Update(); NUnit.Framework.Assert.AreEqual(0, (double)scheduler.GetQueueManager().GetLeafQueue ("root.parentA.childA1", false).GetFairShare().GetMemory() / nodeCapacity * 100, 0); NUnit.Framework.Assert.AreEqual(100, (double)scheduler.GetQueueManager().GetLeafQueue ("root.parentA.childA2", false).GetFairShare().GetMemory() / nodeCapacity * 100, 0.1); VerifySteadyFairShareMemory(scheduler.GetQueueManager().GetLeafQueues(), nodeCapacity ); }
public override void Handle(SchedulerEvent @event) { switch (@event.GetType()) { case SchedulerEventType.NodeAdded: { NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)@event; AddNode(nodeAddedEvent.GetAddedRMNode()); RecoverContainersOnNode(nodeAddedEvent.GetContainerReports(), nodeAddedEvent.GetAddedRMNode ()); break; } case SchedulerEventType.NodeRemoved: { NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)@event; RemoveNode(nodeRemovedEvent.GetRemovedRMNode()); break; } case SchedulerEventType.NodeResourceUpdate: { NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = (NodeResourceUpdateSchedulerEvent )@event; UpdateNodeResource(nodeResourceUpdatedEvent.GetRMNode(), nodeResourceUpdatedEvent .GetResourceOption()); break; } case SchedulerEventType.NodeUpdate: { NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)@event; NodeUpdate(nodeUpdatedEvent.GetRMNode()); break; } case SchedulerEventType.AppAdded: { AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)@event; AddApplication(appAddedEvent.GetApplicationId(), appAddedEvent.GetQueue(), appAddedEvent .GetUser(), appAddedEvent.GetIsAppRecovering()); break; } case SchedulerEventType.AppRemoved: { AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)@event; DoneApplication(appRemovedEvent.GetApplicationID(), appRemovedEvent.GetFinalState ()); break; } case SchedulerEventType.AppAttemptAdded: { AppAttemptAddedSchedulerEvent appAttemptAddedEvent = (AppAttemptAddedSchedulerEvent )@event; AddApplicationAttempt(appAttemptAddedEvent.GetApplicationAttemptId(), appAttemptAddedEvent .GetTransferStateFromPreviousAttempt(), appAttemptAddedEvent.GetIsAttemptRecovering ()); break; } case SchedulerEventType.AppAttemptRemoved: { AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = (AppAttemptRemovedSchedulerEvent )@event; try { DoneApplicationAttempt(appAttemptRemovedEvent.GetApplicationAttemptID(), appAttemptRemovedEvent .GetFinalAttemptState(), appAttemptRemovedEvent.GetKeepContainersAcrossAppAttempts ()); } catch (IOException ie) { Log.Error("Unable to remove application " + appAttemptRemovedEvent.GetApplicationAttemptID (), ie); } break; } case SchedulerEventType.ContainerExpired: { ContainerExpiredSchedulerEvent containerExpiredEvent = (ContainerExpiredSchedulerEvent )@event; ContainerId containerid = containerExpiredEvent.GetContainerId(); CompletedContainer(GetRMContainer(containerid), SchedulerUtils.CreateAbnormalContainerStatus (containerid, SchedulerUtils.ExpiredContainer), RMContainerEventType.Expire); break; } case SchedulerEventType.ContainerRescheduled: { ContainerRescheduledEvent containerRescheduledEvent = (ContainerRescheduledEvent) @event; RMContainer container = containerRescheduledEvent.GetContainer(); RecoverResourceRequestForContainer(container); break; } default: { Log.Error("Invalid eventtype " + @event.GetType() + ". Ignoring!"); break; } } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException /// "/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> protected internal virtual void TestPlanFollower(bool isMove) { // Initialize plan based on move flag plan = new InMemoryPlan(scheduler.GetRootQueueMetrics(), policy, mAgent, scheduler .GetClusterResource(), 1L, res, scheduler.GetMinimumResourceCapability(), maxAlloc , "dedicated", null, isMove); // add a few reservations to the plan long ts = Runtime.CurrentTimeMillis(); ReservationId r1 = ReservationId.NewInstance(ts, 1); int[] f1 = new int[] { 10, 10, 10, 10, 10 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r1, null, "u3", "dedicated", 0, 0 + f1.Length, ReservationSystemTestUtil.GenerateAllocation (0L, 1L, f1), res, minAlloc))); ReservationId r2 = ReservationId.NewInstance(ts, 2); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r2, null, "u3", "dedicated", 3, 3 + f1.Length, ReservationSystemTestUtil.GenerateAllocation (3L, 1L, f1), res, minAlloc))); ReservationId r3 = ReservationId.NewInstance(ts, 3); int[] f2 = new int[] { 0, 10, 20, 10, 0 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r3, null, "u4", "dedicated", 10, 10 + f2.Length, ReservationSystemTestUtil.GenerateAllocation (10L, 1L, f2), res, minAlloc))); AbstractSchedulerPlanFollower planFollower = CreatePlanFollower(); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(0L); planFollower.Run(); Queue q = GetReservationQueue(r1.ToString()); AssertReservationQueueExists(r1); // submit an app to r1 string user_0 = "test-user"; ApplicationId appId = ApplicationId.NewInstance(0, 1); ApplicationAttemptId appAttemptId_0 = ApplicationAttemptId.NewInstance(appId, 0); AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, q.GetQueueName (), user_0); scheduler.Handle(addAppEvent); AppAttemptAddedSchedulerEvent appAttemptAddedEvent = new AppAttemptAddedSchedulerEvent (appAttemptId_0, false); scheduler.Handle(appAttemptAddedEvent); // initial default reservation queue should have no apps Queue defQ = GetDefaultQueue(); NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); AssertReservationQueueExists(r1, 0.1, 0.1); NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(q)); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueDoesNotExist(r3); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(3L); planFollower.Run(); NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); AssertReservationQueueExists(r1, 0.1, 0.1); NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(q)); AssertReservationQueueExists(r2, 0.1, 0.1); AssertReservationQueueDoesNotExist(r3); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(10L); planFollower.Run(); q = GetReservationQueue(r1.ToString()); if (isMove) { // app should have been moved to default reservation queue NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(defQ)); NUnit.Framework.Assert.IsNull(q); } else { // app should be killed NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); NUnit.Framework.Assert.IsNotNull(q); AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = new AppAttemptRemovedSchedulerEvent (appAttemptId_0, RMAppAttemptState.Killed, false); scheduler.Handle(appAttemptRemovedEvent); } AssertReservationQueueDoesNotExist(r2); AssertReservationQueueExists(r3, 0, 1.0); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(11L); planFollower.Run(); if (isMove) { // app should have been moved to default reservation queue NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(defQ)); } else { // app should be killed NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); } AssertReservationQueueDoesNotExist(r1); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueExists(r3, 0.1, 0.1); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(12L); planFollower.Run(); AssertReservationQueueDoesNotExist(r1); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueExists(r3, 0.2, 0.2); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(16L); planFollower.Run(); AssertReservationQueueDoesNotExist(r1); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueDoesNotExist(r3); VerifyCapacity(defQ); }
public virtual void TestResourceAllocation() { Log.Info("--- START: testResourceAllocation ---"); int memory = 4 * 1024; int vcores = 4; // Register node1 string host1 = "host1"; NodeManager nm1 = RegisterNode(host1, 1234, 2345, NetworkTopology.DefaultRack, Resources .CreateResource(memory, vcores)); // Register node2 string host2 = "host2"; NodeManager nm2 = RegisterNode(host2, 1234, 2345, NetworkTopology.DefaultRack, Resources .CreateResource(memory / 2, vcores / 2)); // Submit an application Application application = new Application("user1", resourceManager); application.Submit(); application.AddNodeManager(host1, 1234, nm1); application.AddNodeManager(host2, 1234, nm2); // Application resource requirements int memory1 = 1024; Org.Apache.Hadoop.Yarn.Api.Records.Resource capability1 = Resources.CreateResource (memory1, 1); Priority priority1 = Priority.Create(1); application.AddResourceRequestSpec(priority1, capability1); Task t1 = new Task(application, priority1, new string[] { host1, host2 }); application.AddTask(t1); int memory2 = 2048; Org.Apache.Hadoop.Yarn.Api.Records.Resource capability2 = Resources.CreateResource (memory2, 1); Priority priority0 = Priority.Create(0); // higher application.AddResourceRequestSpec(priority0, capability2); // Send resource requests to the scheduler application.Schedule(); // Send a heartbeat to kick the tires on the Scheduler NodeUpdate(nm1); // Get allocations from the scheduler application.Schedule(); CheckResourceUsage(nm1, nm2); Log.Info("Adding new tasks..."); Task t2 = new Task(application, priority1, new string[] { host1, host2 }); application.AddTask(t2); Task t3 = new Task(application, priority0, new string[] { ResourceRequest.Any }); application.AddTask(t3); // Send resource requests to the scheduler application.Schedule(); CheckResourceUsage(nm1, nm2); // Send heartbeats to kick the tires on the Scheduler NodeUpdate(nm2); NodeUpdate(nm2); NodeUpdate(nm1); NodeUpdate(nm1); // Get allocations from the scheduler Log.Info("Trying to allocate..."); application.Schedule(); CheckResourceUsage(nm1, nm2); // Complete tasks Log.Info("Finishing up tasks..."); application.FinishTask(t1); application.FinishTask(t2); application.FinishTask(t3); // Notify scheduler application is finished. AppAttemptRemovedSchedulerEvent appRemovedEvent1 = new AppAttemptRemovedSchedulerEvent (application.GetApplicationAttemptId(), RMAppAttemptState.Finished, false); resourceManager.GetResourceScheduler().Handle(appRemovedEvent1); CheckResourceUsage(nm1, nm2); Log.Info("--- END: testResourceAllocation ---"); }