public virtual void TestDefaultSingleQueueMetrics() { string queueName = "single"; string user = "******"; QueueMetrics metrics = QueueMetrics.ForQueue(ms, queueName, null, false, conf); MetricsSource queueSource = QueueSource(ms, queueName); AppSchedulingInfo app = MockApp(user); metrics.SubmitApp(user); MetricsSource userSource = UserSource(ms, queueName, user); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.SubmitAppAttempt(user); CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100)); metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0 ); metrics.RunAppAttempt(app.GetApplicationId(), user); CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true); metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true); CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0 , 0); metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2)); CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0 , 0); metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser()); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.FinishApp(user, RMAppState.Finished); CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true); NUnit.Framework.Assert.IsNull(userSource); }
public virtual void Move(Queue newQueue) { lock (this) { QueueMetrics oldMetrics = queue.GetMetrics(); QueueMetrics newMetrics = newQueue.GetMetrics(); string user = GetUser(); foreach (RMContainer liveContainer in liveContainers.Values) { Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = liveContainer.GetContainer ().GetResource(); oldMetrics.ReleaseResources(user, 1, resource); newMetrics.AllocateResources(user, 1, resource, false); } foreach (IDictionary <NodeId, RMContainer> map in reservedContainers.Values) { foreach (RMContainer reservedContainer in map.Values) { Org.Apache.Hadoop.Yarn.Api.Records.Resource resource = reservedContainer.GetReservedResource (); oldMetrics.UnreserveResource(user, resource); newMetrics.ReserveResource(user, resource); } } appSchedulingInfo.Move(newQueue); this.queue = newQueue; } }
public virtual void TestMetricsCache() { MetricsSystem ms = new MetricsSystemImpl("cache"); ms.Start(); try { string p1 = "root1"; string leafQueueName = "root1.leaf"; QueueMetrics p1Metrics = QueueMetrics.ForQueue(ms, p1, null, true, conf); Queue parentQueue1 = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(p1Metrics ).from.GetMetrics()); QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue1, true , conf); NUnit.Framework.Assert.IsNotNull("QueueMetrics for A shoudn't be null", metrics); // Re-register to check for cache hit, shouldn't blow up metrics-system... // also, verify parent-metrics QueueMetrics alterMetrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue1 , true, conf); NUnit.Framework.Assert.IsNotNull("QueueMetrics for alterMetrics shoudn't be null" , alterMetrics); } finally { ms.Shutdown(); } }
public virtual void Move(Queue newQueue) { lock (this) { QueueMetrics oldMetrics = queue.GetMetrics(); QueueMetrics newMetrics = newQueue.GetMetrics(); foreach (IDictionary <string, ResourceRequest> asks in requests.Values) { ResourceRequest request = asks[ResourceRequest.Any]; if (request != null) { oldMetrics.DecrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); newMetrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); } } oldMetrics.MoveAppFrom(this); newMetrics.MoveAppTo(this); activeUsersManager.DeactivateApplication(user, applicationId); activeUsersManager = newQueue.GetActiveUsersManager(); activeUsersManager.ActivateApplication(user, applicationId); this.queue = newQueue; this.queueName = newQueue.GetQueueName(); } }
public static MetricsSource UserSource(MetricsSystem ms, string queue, string user ) { MetricsSource s = ms.GetSource(QueueMetrics.SourceName(queue).Append(",user=").Append (user).ToString()); return(s); }
private Queue CreateQueue(string name, Queue parent) { QueueMetrics metrics = QueueMetrics.ForQueue(name, parent, false, conf); ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics); Queue queue = Org.Mockito.Mockito.Mock <Queue>(); Org.Mockito.Mockito.When(queue.GetMetrics()).ThenReturn(metrics); Org.Mockito.Mockito.When(queue.GetActiveUsersManager()).ThenReturn(activeUsersManager ); return(queue); }
public virtual void TestMove() { string user = "******"; Queue parentQueue = CreateQueue("parent", null); Queue oldQueue = CreateQueue("old", parentQueue); Queue newQueue = CreateQueue("new", parentQueue); QueueMetrics parentMetrics = parentQueue.GetMetrics(); QueueMetrics oldMetrics = oldQueue.GetMetrics(); QueueMetrics newMetrics = newQueue.GetMetrics(); ApplicationAttemptId appAttId = CreateAppAttemptId(0, 0); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetEpoch()).ThenReturn(3L); SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, oldQueue, oldQueue.GetActiveUsersManager(), rmContext); oldMetrics.SubmitApp(user); // confirm that containerId is calculated based on epoch. NUnit.Framework.Assert.AreEqual(unchecked ((long)(0x30000000001L)), app.GetNewContainerId ()); // Resource request Resource requestedResource = Resource.NewInstance(1536, 2); Priority requestedPriority = Priority.NewInstance(2); ResourceRequest request = ResourceRequest.NewInstance(requestedPriority, ResourceRequest .Any, requestedResource, 3); app.UpdateResourceRequests(Arrays.AsList(request)); // Allocated container RMContainer container1 = CreateRMContainer(appAttId, 1, requestedResource); app.liveContainers[container1.GetContainerId()] = container1; SchedulerNode node = CreateNode(); app.appSchedulingInfo.Allocate(NodeType.OffSwitch, node, requestedPriority, request , container1.GetContainer()); // Reserved container Priority prio1 = Priority.NewInstance(1); Resource reservedResource = Resource.NewInstance(2048, 3); RMContainer container2 = CreateReservedRMContainer(appAttId, 1, reservedResource, node.GetNodeID(), prio1); IDictionary <NodeId, RMContainer> reservations = new Dictionary <NodeId, RMContainer >(); reservations[node.GetNodeID()] = container2; app.reservedContainers[prio1] = reservations; oldMetrics.ReserveResource(user, reservedResource); CheckQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); CheckQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0); CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); app.Move(newQueue); CheckQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0); CheckQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); }
public virtual void TestMetricsInitializedOnRMInit() { YarnConfiguration conf = new YarnConfiguration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); QueueMetrics metrics = rm.GetResourceScheduler().GetRootQueueMetrics(); CheckApps(metrics, 0, 0, 0, 0, 0, 0, true); MetricsAsserts.AssertGauge("ReservedContainers", 0, metrics); }
private void CheckQueueMetrics(QueueMetrics metrics, int activeApps, int runningApps , int allocMb, int allocVcores, int reservedMb, int reservedVcores, int pendingMb , int pendingVcores) { NUnit.Framework.Assert.AreEqual(activeApps, metrics.GetActiveApps()); NUnit.Framework.Assert.AreEqual(runningApps, metrics.GetAppsRunning()); NUnit.Framework.Assert.AreEqual(allocMb, metrics.GetAllocatedMB()); NUnit.Framework.Assert.AreEqual(allocVcores, metrics.GetAllocatedVirtualCores()); NUnit.Framework.Assert.AreEqual(reservedMb, metrics.GetReservedMB()); NUnit.Framework.Assert.AreEqual(reservedVcores, metrics.GetReservedVirtualCores() ); NUnit.Framework.Assert.AreEqual(pendingMb, metrics.GetPendingMB()); NUnit.Framework.Assert.AreEqual(pendingVcores, metrics.GetPendingVirtualCores()); }
public virtual void Stop(RMAppAttemptState rmAppAttemptFinalState) { lock (this) { // clear pending resources metrics for the application QueueMetrics metrics = queue.GetMetrics(); foreach (IDictionary <string, ResourceRequest> asks in requests.Values) { ResourceRequest request = asks[ResourceRequest.Any]; if (request != null) { metrics.DecrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); } } metrics.FinishAppAttempt(applicationId, pending, user); // Clear requests themselves ClearRequests(); } }
public virtual void RecoverContainer(RMContainer rmContainer) { lock (this) { QueueMetrics metrics = queue.GetMetrics(); if (pending) { // If there was any container to recover, the application was // running from scheduler's POV. pending = false; metrics.RunAppAttempt(applicationId, user); } // Container is completed. Skip recovering resources. if (rmContainer.GetState().Equals(RMContainerState.Completed)) { return; } metrics.AllocateResources(user, 1, rmContainer.GetAllocatedResource(), false); } }
public virtual void TestQueueAppMetricsForMultipleFailures() { string queueName = "single"; string user = "******"; QueueMetrics metrics = QueueMetrics.ForQueue(ms, queueName, null, false, new Configuration ()); MetricsSource queueSource = QueueSource(ms, queueName); AppSchedulingInfo app = MockApp(user); metrics.SubmitApp(user); MetricsSource userSource = UserSource(ms, queueName, user); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.SubmitAppAttempt(user); CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.RunAppAttempt(app.GetApplicationId(), user); CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true); metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser()); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); // As the application has failed, framework retries the same application // based on configuration metrics.SubmitAppAttempt(user); CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.RunAppAttempt(app.GetApplicationId(), user); CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true); // Suppose say application has failed this time as well. metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser()); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); // As the application has failed, framework retries the same application // based on configuration metrics.SubmitAppAttempt(user); CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true); metrics.RunAppAttempt(app.GetApplicationId(), user); CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true); // Suppose say application has failed, and there's no more retries. metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser()); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); metrics.FinishApp(user, RMAppState.Failed); CheckApps(queueSource, 1, 0, 0, 0, 1, 0, true); NUnit.Framework.Assert.IsNull(userSource); }
public virtual void TestCollectAllMetrics() { string queueName = "single"; QueueMetrics.ForQueue(ms, queueName, null, false, conf); MetricsSource queueSource = QueueSource(ms, queueName); CheckApps(queueSource, 0, 0, 0, 0, 0, 0, true); try { // do not collect all metrics CheckApps(queueSource, 0, 0, 0, 0, 0, 0, false); NUnit.Framework.Assert.Fail(); } catch (Exception e) { NUnit.Framework.Assert.IsTrue(e.Message.Contains("Expected exactly one metric for name " )); } // collect all metrics CheckApps(queueSource, 0, 0, 0, 0, 0, 0, true); }
/// <summary> /// Resources have been allocated to this application by the resource /// scheduler. /// </summary> /// <remarks> /// Resources have been allocated to this application by the resource /// scheduler. Track them. /// </remarks> /// <param name="type">the type of the node</param> /// <param name="node">the nodeinfo of the node</param> /// <param name="priority">the priority of the request.</param> /// <param name="request">the request</param> /// <param name="container">the containers allocated.</param> public virtual IList <ResourceRequest> Allocate(NodeType type, SchedulerNode node, Priority priority, ResourceRequest request, Container container) { lock (this) { IList <ResourceRequest> resourceRequests = new AList <ResourceRequest>(); if (type == NodeType.NodeLocal) { AllocateNodeLocal(node, priority, request, container, resourceRequests); } else { if (type == NodeType.RackLocal) { AllocateRackLocal(node, priority, request, container, resourceRequests); } else { AllocateOffSwitch(node, priority, request, container, resourceRequests); } } QueueMetrics metrics = queue.GetMetrics(); if (pending) { // once an allocation is done we assume the application is // running from scheduler's POV. pending = false; metrics.RunAppAttempt(applicationId, user); } if (Log.IsDebugEnabled()) { Log.Debug("allocate: applicationId=" + applicationId + " container=" + container. GetId() + " host=" + container.GetNodeId().ToString() + " user="******" resource=" + request.GetCapability()); } metrics.AllocateResources(user, 1, request.GetCapability(), true); return(resourceRequests); } }
public ActiveUsersManager(QueueMetrics metrics) { this.metrics = metrics; }
public virtual void TestTwoLevelWithUserMetrics() { string parentQueueName = "root"; string leafQueueName = "root.leaf"; string user = "******"; QueueMetrics parentMetrics = QueueMetrics.ForQueue(ms, parentQueueName, null, true , conf); Queue parentQueue = MockitoMaker.Make(MockitoMaker.Stub <Queue>().Returning(parentMetrics ).from.GetMetrics()); QueueMetrics metrics = QueueMetrics.ForQueue(ms, leafQueueName, parentQueue, true , conf); MetricsSource parentQueueSource = QueueSource(ms, parentQueueName); MetricsSource queueSource = QueueSource(ms, leafQueueName); AppSchedulingInfo app = MockApp(user); metrics.SubmitApp(user); MetricsSource userSource = UserSource(ms, leafQueueName, user); MetricsSource parentUserSource = UserSource(ms, parentQueueName, user); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(userSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true); metrics.SubmitAppAttempt(user); CheckApps(queueSource, 1, 1, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 1, 0, 0, 0, 0, true); CheckApps(userSource, 1, 1, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 1, 0, 0, 0, 0, true); parentMetrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100 )); metrics.SetAvailableResourcesToQueue(Resources.CreateResource(100 * Gb, 100)); parentMetrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10)); metrics.SetAvailableResourcesToUser(user, Resources.CreateResource(10 * Gb, 10)); metrics.IncrPendingResources(user, 5, Resources.CreateResource(3 * Gb, 3)); CheckResources(queueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0, 0, 0 ); CheckResources(parentQueueSource, 0, 0, 0, 0, 0, 100 * Gb, 100, 15 * Gb, 15, 5, 0 , 0, 0); CheckResources(userSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0, 0); CheckResources(parentUserSource, 0, 0, 0, 0, 0, 10 * Gb, 10, 15 * Gb, 15, 5, 0, 0 , 0); metrics.RunAppAttempt(app.GetApplicationId(), user); CheckApps(queueSource, 1, 0, 1, 0, 0, 0, true); CheckApps(userSource, 1, 0, 1, 0, 0, 0, true); metrics.AllocateResources(user, 3, Resources.CreateResource(2 * Gb, 2), true); metrics.ReserveResource(user, Resources.CreateResource(3 * Gb, 3)); // Available resources is set externally, as it depends on dynamic // configurable cluster/queue resources CheckResources(queueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2, 3 * Gb, 3, 1); CheckResources(parentQueueSource, 6 * Gb, 6, 3, 3, 0, 100 * Gb, 100, 9 * Gb, 9, 2 , 3 * Gb, 3, 1); CheckResources(userSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb, 3, 1); CheckResources(parentUserSource, 6 * Gb, 6, 3, 3, 0, 10 * Gb, 10, 9 * Gb, 9, 2, 3 * Gb, 3, 1); metrics.ReleaseResources(user, 1, Resources.CreateResource(2 * Gb, 2)); metrics.UnreserveResource(user, Resources.CreateResource(3 * Gb, 3)); CheckResources(queueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2, 0, 0 , 0); CheckResources(parentQueueSource, 4 * Gb, 4, 2, 3, 1, 100 * Gb, 100, 9 * Gb, 9, 2 , 0, 0, 0); CheckResources(userSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0, 0, 0 ); CheckResources(parentUserSource, 4 * Gb, 4, 2, 3, 1, 10 * Gb, 10, 9 * Gb, 9, 2, 0 , 0, 0); metrics.FinishAppAttempt(app.GetApplicationId(), app.IsPending(), app.GetUser()); CheckApps(queueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 0, 0, 0, true); CheckApps(userSource, 1, 0, 0, 0, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 0, 0, 0, true); metrics.FinishApp(user, RMAppState.Finished); CheckApps(queueSource, 1, 0, 0, 1, 0, 0, true); CheckApps(parentQueueSource, 1, 0, 0, 1, 0, 0, true); CheckApps(userSource, 1, 0, 0, 1, 0, 0, true); CheckApps(parentUserSource, 1, 0, 0, 1, 0, 0, true); }
public virtual void TearDown() { QueueMetrics.ClearQueueMetrics(); DefaultMetricsSystem.Shutdown(); }
public virtual void SetUp() { ms = new MetricsSystemImpl(); QueueMetrics.ClearQueueMetrics(); }
public static MetricsSource QueueSource(MetricsSystem ms, string queue) { MetricsSource s = ms.GetSource(QueueMetrics.SourceName(queue).ToString()); return(s); }
/// <summary> /// The ApplicationMaster is updating resource requirements for the /// application, by asking for more resources and releasing resources acquired /// by the application. /// </summary> /// <param name="requests">resources to be acquired</param> /// <param name="recoverPreemptedRequest">recover Resource Request on preemption</param> public virtual void UpdateResourceRequests(IList <ResourceRequest> requests, bool recoverPreemptedRequest) { lock (this) { QueueMetrics metrics = queue.GetMetrics(); // Update resource requests foreach (ResourceRequest request in requests) { Priority priority = request.GetPriority(); string resourceName = request.GetResourceName(); bool updatePendingResources = false; ResourceRequest lastRequest = null; if (resourceName.Equals(ResourceRequest.Any)) { if (Log.IsDebugEnabled()) { Log.Debug("update:" + " application=" + applicationId + " request=" + request); } updatePendingResources = true; // Premature optimization? // Assumes that we won't see more than one priority request updated // in one call, reasonable assumption... however, it's totally safe // to activate same application more than once. // Thus we don't need another loop ala the one in decrementOutstanding() // which is needed during deactivate. if (request.GetNumContainers() > 0) { activeUsersManager.ActivateApplication(user, applicationId); } } IDictionary <string, ResourceRequest> asks = this.requests[priority]; if (asks == null) { asks = new ConcurrentHashMap <string, ResourceRequest>(); this.requests[priority] = asks; this.priorities.AddItem(priority); } lastRequest = asks[resourceName]; if (recoverPreemptedRequest && lastRequest != null) { // Increment the number of containers to 1, as it is recovering a // single container. request.SetNumContainers(lastRequest.GetNumContainers() + 1); } asks[resourceName] = request; if (updatePendingResources) { // Similarly, deactivate application? if (request.GetNumContainers() <= 0) { Log.Info("checking for deactivate of application :" + this.applicationId); CheckForDeactivation(); } int lastRequestContainers = lastRequest != null?lastRequest.GetNumContainers() : 0; Org.Apache.Hadoop.Yarn.Api.Records.Resource lastRequestCapability = lastRequest != null?lastRequest.GetCapability() : Resources.None(); metrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); metrics.DecrPendingResources(user, lastRequestContainers, lastRequestCapability); } } } }