public virtual void TestDefaultSingleQueueMetrics() { string queueName = "single"; string user = "******"; QueueMetrics metrics = QueueMetrics.ForQueue(ms, queueName, null, false, conf); MetricsSource queueSource = QueueSource(ms, queueName); AppSchedulingInfo app = MockApp(user); metrics.SubmitApp(user); MetricsSource userSource = UserSource(ms, queueName, user); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.SubmitAppAttempt(user); CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100)); metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0 ); metrics.RunAppAttempt(app.GetApplicationId(), user); CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true); metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true); CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0 , 0); metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2)); CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0 , 0); metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser()); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.FinishApp(user, RMAppState.Finished); CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true); NUnit.Framework.Assert.IsNull(userSource); }
public virtual void Move(Queue newQueue) { lock (this) { QueueMetrics oldMetrics = queue.GetMetrics(); QueueMetrics newMetrics = newQueue.GetMetrics(); string user = GetUser(); foreach (RMContainer liveContainer in liveContainers.Values) { Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = liveContainer.GetContainer ().GetResource(); oldMetrics.ReleaseResources(user, 1, resource); newMetrics.AllocateResources(user, 1, resource, false); } foreach (IDictionary <NodeId, RMContainer> map in reservedContainers.Values) { foreach (RMContainer reservedContainer in map.Values) { Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = reservedContainer.GetReservedResource (); oldMetrics.UnreserveResource(user, resource); newMetrics.ReserveResource(user, resource); } } appSchedulingInfo.Move(newQueue); this.queue = newQueue; } }
public virtual void RecoverContainer(RMContainer rmContainer) { lock (this) { QueueMetrics metrics = queue.GetMetrics(); if (pending) { // If there was any container to recover, the application was // running from scheduler's POV. pending = false; metrics.RunAppAttempt(applicationId, user); } // Container is completed. Skip recovering resources. if (rmContainer.GetState().Equals(RMContainerState.Completed)) { return; } metrics.AllocateResources(user, 1, rmContainer.GetAllocatedResource(), false); } }
/// <summary> /// Resources have been allocated to this application by the resource /// scheduler. /// </summary> /// <remarks> /// Resources have been allocated to this application by the resource /// scheduler. Track them. /// </remarks> /// <param name="type">the type of the node</param> /// <param name="node">the nodeinfo of the node</param> /// <param name="priority">the priority of the request.</param> /// <param name="request">the request</param> /// <param name="container">the containers allocated.</param> public virtual IList <ResourceRequest> Allocate(NodeType type, SchedulerNode node, Priority priority, ResourceRequest request, Container container) { lock (this) { IList <ResourceRequest> resourceRequests = new AList <ResourceRequest>(); if (type == NodeType.NodeLocal) { AllocateNodeLocal(node, priority, request, container, resourceRequests); } else { if (type == NodeType.RackLocal) { AllocateRackLocal(node, priority, request, container, resourceRequests); } else { AllocateOffSwitch(node, priority, request, container, resourceRequests); } } QueueMetrics metrics = queue.GetMetrics(); if (pending) { // once an allocation is done we assume the application is // running from scheduler's POV. pending = false; metrics.RunAppAttempt(applicationId, user); } if (Log.IsDebugEnabled()) { Log.Debug("allocate: applicationId=" + applicationId + " container=" + container. GetId() + " host=" + container.GetNodeId().ToString() + " user="******" resource=" + request.GetCapability()); } metrics.AllocateResources(user, 1, request.GetCapability(), true); return(resourceRequests); } }
public virtual void TestTwoLevelWithUserMetrics() { string parentQueueName = "root"; string leafQueueName = "root.leaf"; string user = "******"; QueueMetrics parentMetrics = QueueMetrics.ForQueue(ms, parentQueueName, null, true , conf); Queue parentQueue = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(parentMetrics ).from.GetMetrics()); QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue, true , conf); MetricsSource parentQueueSource = QueueSource(ms, parentQueueName); MetricsSource queueSource = QueueSource(ms, leafQueueName); AppSchedulingInfo app = MockApp(user); metrics.SubmitApp(user); MetricsSource userSource = UserSource(ms, leafQueueName, user); MetricsSource parentUserSource = UserSource(ms, parentQueueName, user); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(userSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true); metrics.SubmitAppAttempt(user); CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true); CheckApps(userSource, 1, 1, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 1, 0, 0, 0, 0, true); parentMetrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100 )); metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100)); parentMetrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10)); metrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10)); metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3)); CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0 ); CheckResources(parentQueueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0 , 0, 0); CheckResources(userSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0, 0); CheckResources(parentUserSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0 , 0); metrics.RunAppAttempt(app.GetApplicationId(), user); CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true); CheckApps(userSource, 1, 0, 1, 0, 0, 0, true); metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true); metrics.ReserveResource(user, Resources.CreateResource(3 * Gb, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 3 * Gb, 3, 1); CheckResources(parentQueueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2 , 3 * Gb, 3, 1); CheckResources(userSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb, 3, 1); CheckResources(parentUserSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb, 3, 1); metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2)); metrics.UnreserveResource(user, Resources.CreateResource(3 * Gb, 3)); CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0 , 0); CheckResources(parentQueueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2 , 0, 0, 0); CheckResources(userSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0, 0, 0 ); CheckResources(parentUserSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0 , 0, 0); metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser()); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(userSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true); metrics.FinishApp(user, RMAppState.Finished); CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true); CheckApps(userSource, 1, 0, 0, 1, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 1, 0, 0, true); }