public virtual void Move(Queue newQueue) { lock (this) { QueueMetrics oldMetrics = queue.GetMetrics(); QueueMetrics newMetrics = newQueue.GetMetrics(); string user = GetUser(); foreach (RMContainer liveContainer in liveContainers.Values) { Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = liveContainer.GetContainer ().GetResource(); oldMetrics.ReleaseResources(user, 1, resource); newMetrics.AllocateResources(user, 1, resource, false); } foreach (IDictionary <NodeId, RMContainer> map in reservedContainers.Values) { foreach (RMContainer reservedContainer in map.Values) { Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = reservedContainer.GetReservedResource (); oldMetrics.UnreserveResource(user, resource); newMetrics.ReserveResource(user, resource); } } appSchedulingInfo.Move(newQueue); this.queue = newQueue; } }
public virtual void TestMove() { string user = "******"; Queue parentQueue = CreateQueue("parent", null); Queue oldQueue = CreateQueue("old", parentQueue); Queue newQueue = CreateQueue("new", parentQueue); QueueMetrics parentMetrics = parentQueue.GetMetrics(); QueueMetrics oldMetrics = oldQueue.GetMetrics(); QueueMetrics newMetrics = newQueue.GetMetrics(); ApplicationAttemptId appAttId = CreateAppAttemptId(0, 0); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetEpoch()).ThenReturn(3L); SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, oldQueue, oldQueue.GetActiveUsersManager(), rmContext); oldMetrics.SubmitApp(user); // confirm that containerId is calculated based on epoch. NUnit.Framework.Assert.AreEqual(unchecked ((long)(0x30000000001L)), app.GetNewContainerId ()); // Resource request Resource requestedResource = Resource.NewInstance(1536, 2); Priority requestedPriority = Priority.NewInstance(2); ResourceRequest request = ResourceRequest.NewInstance(requestedPriority, ResourceRequest .Any, requestedResource, 3); app.UpdateResourceRequests(Arrays.AsList(request)); // Allocated container RMContainer container1 = CreateRMContainer(appAttId, 1, requestedResource); app.liveContainers[container1.GetContainerId()] = container1; SchedulerNode node = CreateNode(); app.appSchedulingInfo.Allocate(NodeType.OffSwitch, node, requestedPriority, request , container1.GetContainer()); // Reserved container Priority prio1 = Priority.NewInstance(1); Resource reservedResource = Resource.NewInstance(2048, 3); RMContainer container2 = CreateReservedRMContainer(appAttId, 1, reservedResource, node.GetNodeID(), prio1); IDictionary <NodeId, RMContainer> reservations = new Dictionary <NodeId, RMContainer >(); reservations[node.GetNodeID()] = container2; app.reservedContainers[prio1] = reservations; oldMetrics.ReserveResource(user, reservedResource); CheckQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); CheckQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0); CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); app.Move(newQueue); CheckQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0); CheckQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); }
public virtual void TestTwoLevelWithUserMetrics() { string parentQueueName = "root"; string leafQueueName = "root.leaf"; string user = "******"; QueueMetrics parentMetrics = QueueMetrics.ForQueue(ms, parentQueueName, null, true , conf); Queue parentQueue = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(parentMetrics ).from.GetMetrics()); QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue, true , conf); MetricsSource parentQueueSource = QueueSource(ms, parentQueueName); MetricsSource queueSource = QueueSource(ms, leafQueueName); AppSchedulingInfo app = MockApp(user); metrics.SubmitApp(user); MetricsSource userSource = UserSource(ms, leafQueueName, user); MetricsSource parentUserSource = UserSource(ms, parentQueueName, user); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(userSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true); metrics.SubmitAppAttempt(user); CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true); CheckApps(userSource, 1, 1, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 1, 0, 0, 0, 0, true); parentMetrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100 )); metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100)); parentMetrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10)); metrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10)); metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3)); CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0 ); CheckResources(parentQueueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0 , 0, 0); CheckResources(userSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0, 0); CheckResources(parentUserSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0 , 0); metrics.RunAppAttempt(app.GetApplicationId(), user); CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true); CheckApps(userSource, 1, 0, 1, 0, 0, 0, true); metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true); metrics.ReserveResource(user, Resources.CreateResource(3 * Gb, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 3 * Gb, 3, 1); CheckResources(parentQueueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2 , 3 * Gb, 3, 1); CheckResources(userSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb, 3, 1); CheckResources(parentUserSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb, 3, 1); metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2)); metrics.UnreserveResource(user, Resources.CreateResource(3 * Gb, 3)); CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0 , 0); CheckResources(parentQueueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2 , 0, 0, 0); CheckResources(userSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0, 0, 0 ); CheckResources(parentUserSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0 , 0, 0); metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser()); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(userSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true); metrics.FinishApp(user, RMAppState.Finished); CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true); CheckApps(userSource, 1, 0, 0, 1, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 1, 0, 0, true); }