public virtual void Move(Queue newQueue) { lock (this) { QueueMetrics oldMetrics = queue.GetMetrics(); QueueMetrics newMetrics = newQueue.GetMetrics(); foreach (IDictionary <string, ResourceRequest> asks in requests.Values) { ResourceRequest request = asks[ResourceRequest.Any]; if (request != null) { oldMetrics.DecrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); newMetrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); } } oldMetrics.MoveAppFrom(this); newMetrics.MoveAppTo(this); activeUsersManager.DeactivateApplication(user, applicationId); activeUsersManager = newQueue.GetActiveUsersManager(); activeUsersManager.ActivateApplication(user, applicationId); this.queue = newQueue; this.queueName = newQueue.GetQueueName(); } }
public virtual void TestDefaultSingleQueueMetrics() { string queueName = "single"; string user = "******"; QueueMetrics metrics = QueueMetrics.ForQueue(ms, queueName, null, false, conf); MetricsSource queueSource = QueueSource(ms, queueName); AppSchedulingInfo app = MockApp(user); metrics.SubmitApp(user); MetricsSource userSource = UserSource(ms, queueName, user); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.SubmitAppAttempt(user); CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100)); metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0 ); metrics.RunAppAttempt(app.GetApplicationId(), user); CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true); metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true); CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0 , 0); metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2)); CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0 , 0); metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser()); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.FinishApp(user, RMAppState.Finished); CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true); NUnit.Framework.Assert.IsNull(userSource); }
/// <summary> /// The ApplicationMaster is updating resource requirements for the /// application, by asking for more resources and releasing resources acquired /// by the application. /// </summary> /// <param name="requests">resources to be acquired</param> /// <param name="recoverPreemptedRequest">recover Resource Request on preemption</param> public virtual void UpdateResourceRequests(IList <ResourceRequest> requests, bool recoverPreemptedRequest) { lock (this) { QueueMetrics metrics = queue.GetMetrics(); // Update resource requests foreach (ResourceRequest request in requests) { Priority priority = request.GetPriority(); string resourceName = request.GetResourceName(); bool updatePendingResources = false; ResourceRequest lastRequest = null; if (resourceName.Equals(ResourceRequest.Any)) { if (Log.IsDebugEnabled()) { Log.Debug("update:" + " application=" + applicationId + " request=" + request); } updatePendingResources = true; // Premature optimization? // Assumes that we won't see more than one priority request updated // in one call, reasonable assumption... however, it's totally safe // to activate same application more than once. // Thus we don't need another loop ala the one in decrementOutstanding() // which is needed during deactivate. if (request.GetNumContainers() > 0) { activeUsersManager.ActivateApplication(user, applicationId); } } IDictionary <string, ResourceRequest> asks = this.requests[priority]; if (asks == null) { asks = new ConcurrentHashMap <string, ResourceRequest>(); this.requests[priority] = asks; this.priorities.AddItem(priority); } lastRequest = asks[resourceName]; if (recoverPreemptedRequest && lastRequest != null) { // Increment the number of containers to 1, as it is recovering a // single container. request.SetNumContainers(lastRequest.GetNumContainers() + 1); } asks[resourceName] = request; if (updatePendingResources) { // Similarly, deactivate application? if (request.GetNumContainers() <= 0) { Log.Info("checking for deactivate of application :" + this.applicationId); CheckForDeactivation(); } int lastRequestContainers = lastRequest != null?lastRequest.GetNumContainers() : 0; Org.Apache.Hadoop.Yarn.Api.Records.Resource lastRequestCapability = lastRequest != null?lastRequest.GetCapability() : Resources.None(); metrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); metrics.DecrPendingResources(user, lastRequestContainers, lastRequestCapability); } } } }
public virtual void TestTwoLevelWithUserMetrics() { string parentQueueName = "root"; string leafQueueName = "root.leaf"; string user = "******"; QueueMetrics parentMetrics = QueueMetrics.ForQueue(ms, parentQueueName, null, true , conf); Queue parentQueue = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(parentMetrics ).from.GetMetrics()); QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue, true , conf); MetricsSource parentQueueSource = QueueSource(ms, parentQueueName); MetricsSource queueSource = QueueSource(ms, leafQueueName); AppSchedulingInfo app = MockApp(user); metrics.SubmitApp(user); MetricsSource userSource = UserSource(ms, leafQueueName, user); MetricsSource parentUserSource = UserSource(ms, parentQueueName, user); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(userSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true); metrics.SubmitAppAttempt(user); CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true); CheckApps(userSource, 1, 1, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 1, 0, 0, 0, 0, true); parentMetrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100 )); metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100)); parentMetrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10)); metrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10)); metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3)); CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0 ); CheckResources(parentQueueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0 , 0, 0); CheckResources(userSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0, 0); CheckResources(parentUserSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0 , 0); metrics.RunAppAttempt(app.GetApplicationId(), user); CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true); CheckApps(userSource, 1, 0, 1, 0, 0, 0, true); metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true); metrics.ReserveResource(user, Resources.CreateResource(3 * Gb, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 3 * Gb, 3, 1); CheckResources(parentQueueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2 , 3 * Gb, 3, 1); CheckResources(userSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb, 3, 1); CheckResources(parentUserSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb, 3, 1); metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2)); metrics.UnreserveResource(user, Resources.CreateResource(3 * Gb, 3)); CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0 , 0); CheckResources(parentQueueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2 , 0, 0, 0); CheckResources(userSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0, 0, 0 ); CheckResources(parentUserSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0 , 0, 0); metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser()); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(userSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true); metrics.FinishApp(user, RMAppState.Finished); CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true); CheckApps(userSource, 1, 0, 0, 1, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 1, 0, 0, true); }