public virtual void TestUpdateMaxAllocationUsesTotal() { int configuredMaxVCores = 20; int configuredMaxMemory = 10 * 1024; Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(configuredMaxMemory, configuredMaxVCores); ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores ); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); MockRM rm = new MockRM(conf); try { rm.Start(); AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler( ); Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(0, 0); Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1024, 5); Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2048, 10); SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>(); Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo" , 8080)); Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource ); Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1); SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>(); Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar" , 8081)); Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource ); Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); scheduler.nodes = new Dictionary <NodeId, SchedulerNode>(); scheduler.nodes[mockNode1.GetNodeID()] = mockNode1; scheduler.UpdateMaximumAllocation(mockNode1, true); VerifyMaximumResourceCapability(fullResource1, scheduler); scheduler.nodes[mockNode2.GetNodeID()] = mockNode2; scheduler.UpdateMaximumAllocation(mockNode2, true); VerifyMaximumResourceCapability(fullResource2, scheduler); Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID()); scheduler.UpdateMaximumAllocation(mockNode2, false); VerifyMaximumResourceCapability(fullResource1, scheduler); Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID()); scheduler.UpdateMaximumAllocation(mockNode1, false); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); } finally { rm.Stop(); } }
/// <summary>Process resource update on a node.</summary> public virtual void UpdateNodeResource(RMNode nm, ResourceOption resourceOption) { lock (this) { SchedulerNode node = GetSchedulerNode(nm.GetNodeID()); Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = resourceOption.GetResource (); Org.Apache.Hadoop.Yarn.Api.Records.Resource oldResource = node.GetTotalResource(); if (!oldResource.Equals(newResource)) { // Log resource change Log.Info("Update resource on node: " + node.GetNodeName() + " from: " + oldResource + ", to: " + newResource); Sharpen.Collections.Remove(nodes, nm.GetNodeID()); UpdateMaximumAllocation(node, false); // update resource to node node.SetTotalResource(newResource); nodes[nm.GetNodeID()] = (N)node; UpdateMaximumAllocation(node, true); // update resource to clusterResource Resources.SubtractFrom(clusterResource, oldResource); Resources.AddTo(clusterResource, newResource); } else { // Log resource change Log.Warn("Update resource on node: " + node.GetNodeName() + " with the same resource: " + newResource); } } }
private SchedulerNode CreateNode() { SchedulerNode node = Org.Mockito.Mockito.Mock <SchedulerNode>(); Org.Mockito.Mockito.When(node.GetNodeName()).ThenReturn("somehost"); Org.Mockito.Mockito.When(node.GetRackName()).ThenReturn("somerack"); Org.Mockito.Mockito.When(node.GetNodeID()).ThenReturn(nodeId); return(node); }
public virtual void TestMove() { string user = "******"; Queue parentQueue = CreateQueue("parent", null); Queue oldQueue = CreateQueue("old", parentQueue); Queue newQueue = CreateQueue("new", parentQueue); QueueMetrics parentMetrics = parentQueue.GetMetrics(); QueueMetrics oldMetrics = oldQueue.GetMetrics(); QueueMetrics newMetrics = newQueue.GetMetrics(); ApplicationAttemptId appAttId = CreateAppAttemptId(0, 0); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetEpoch()).ThenReturn(3L); SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user, oldQueue, oldQueue.GetActiveUsersManager(), rmContext); oldMetrics.SubmitApp(user); // confirm that containerId is calculated based on epoch. NUnit.Framework.Assert.AreEqual(unchecked ((long)(0x30000000001L)), app.GetNewContainerId ()); // Resource request Resource requestedResource = Resource.NewInstance(1536, 2); Priority requestedPriority = Priority.NewInstance(2); ResourceRequest request = ResourceRequest.NewInstance(requestedPriority, ResourceRequest .Any, requestedResource, 3); app.UpdateResourceRequests(Arrays.AsList(request)); // Allocated container RMContainer container1 = CreateRMContainer(appAttId, 1, requestedResource); app.liveContainers[container1.GetContainerId()] = container1; SchedulerNode node = CreateNode(); app.appSchedulingInfo.Allocate(NodeType.OffSwitch, node, requestedPriority, request , container1.GetContainer()); // Reserved container Priority prio1 = Priority.NewInstance(1); Resource reservedResource = Resource.NewInstance(2048, 3); RMContainer container2 = CreateReservedRMContainer(appAttId, 1, reservedResource, node.GetNodeID(), prio1); IDictionary <NodeId, RMContainer> reservations = new Dictionary <NodeId, RMContainer >(); reservations[node.GetNodeID()] = container2; app.reservedContainers[prio1] = reservations; oldMetrics.ReserveResource(user, reservedResource); CheckQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); CheckQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0); CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); app.Move(newQueue); CheckQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0); CheckQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4); }
/// <summary> /// The /// <see cref="ResourceScheduler"/> /// is allocating data-local resources to the /// application. /// </summary> /// <param name="allocatedContainers">resources allocated to the application</param> private void AllocateOffSwitch(SchedulerNode node, Priority priority, ResourceRequest offSwitchRequest, Container container, IList <ResourceRequest> resourceRequests) { lock (this) { // Update future requirements DecrementOutstanding(offSwitchRequest); // Update cloned OffRack requests for recovery resourceRequests.AddItem(CloneResourceRequest(offSwitchRequest)); } }
/// <summary> /// Has the application reserved the given <code>node</code> at the /// given <code>priority</code>? /// </summary> /// <param name="node">node to be checked</param> /// <param name="priority">priority of reserved container</param> /// <returns>true is reserved, false if not</returns> public virtual bool IsReserved(SchedulerNode node, Priority priority) { lock (this) { IDictionary <NodeId, RMContainer> reservedContainers = this.reservedContainers[priority ]; if (reservedContainers != null) { return(reservedContainers.Contains(node.GetNodeID())); } return(false); } }
/// <summary> /// The /// <see cref="ResourceScheduler"/> /// is allocating data-local resources to the /// application. /// </summary> /// <param name="allocatedContainers">resources allocated to the application</param> private void AllocateRackLocal(SchedulerNode node, Priority priority, ResourceRequest rackLocalRequest, Container container, IList <ResourceRequest> resourceRequests) { lock (this) { // Update future requirements DecResourceRequest(node.GetRackName(), priority, rackLocalRequest); ResourceRequest offRackRequest = requests[priority][ResourceRequest.Any]; DecrementOutstanding(offRackRequest); // Update cloned RackLocal and OffRack requests for recovery resourceRequests.AddItem(CloneResourceRequest(rackLocalRequest)); resourceRequests.AddItem(CloneResourceRequest(offRackRequest)); } }
protected internal virtual void ContainerLaunchedOnNode(ContainerId containerId, SchedulerNode node) { lock (this) { // Get the application for the finished container SchedulerApplicationAttempt application = GetCurrentAttemptForContainer(containerId ); if (application == null) { Log.Info("Unknown application " + containerId.GetApplicationAttemptId().GetApplicationId () + " launched container " + containerId + " on node: " + node); this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeCleanContainerEvent (node.GetNodeID(), containerId)); return; } application.ContainerLaunchedOnNode(containerId, node.GetNodeID()); } }
/// <summary> /// Resources have been allocated to this application by the resource /// scheduler. /// </summary> /// <remarks> /// Resources have been allocated to this application by the resource /// scheduler. Track them. /// </remarks> /// <param name="type">the type of the node</param> /// <param name="node">the nodeinfo of the node</param> /// <param name="priority">the priority of the request.</param> /// <param name="request">the request</param> /// <param name="container">the containers allocated.</param> public virtual IList <ResourceRequest> Allocate(NodeType type, SchedulerNode node, Priority priority, ResourceRequest request, Container container) { lock (this) { IList <ResourceRequest> resourceRequests = new AList <ResourceRequest>(); if (type == NodeType.NodeLocal) { AllocateNodeLocal(node, priority, request, container, resourceRequests); } else { if (type == NodeType.RackLocal) { AllocateRackLocal(node, priority, request, container, resourceRequests); } else { AllocateOffSwitch(node, priority, request, container, resourceRequests); } } QueueMetrics metrics = queue.GetMetrics(); if (pending) { // once an allocation is done we assume the application is // running from scheduler's POV. pending = false; metrics.RunAppAttempt(applicationId, user); } if (Log.IsDebugEnabled()) { Log.Debug("allocate: applicationId=" + applicationId + " container=" + container. GetId() + " host=" + container.GetNodeId().ToString() + " user="******" resource=" + request.GetCapability()); } metrics.AllocateResources(user, 1, request.GetCapability(), true); return(resourceRequests); } }
public virtual RMContainer Reserve(SchedulerNode node, Priority priority, RMContainer rmContainer, Container container) { lock (this) { // Create RMContainer if necessary if (rmContainer == null) { rmContainer = new RMContainerImpl(container, GetApplicationAttemptId(), node.GetNodeID (), appSchedulingInfo.GetUser(), rmContext); Resources.AddTo(currentReservation, container.GetResource()); // Reset the re-reservation count ResetReReservations(priority); } else { // Note down the re-reservation AddReReservation(priority); } rmContainer.Handle(new RMContainerReservedEvent(container.GetId(), container.GetResource (), node.GetNodeID(), priority)); IDictionary <NodeId, RMContainer> reservedContainers = this.reservedContainers[priority ]; if (reservedContainers == null) { reservedContainers = new Dictionary <NodeId, RMContainer>(); this.reservedContainers[priority] = reservedContainers; } reservedContainers[node.GetNodeID()] = rmContainer; if (Log.IsDebugEnabled()) { Log.Debug("Application attempt " + GetApplicationAttemptId() + " reserved container " + rmContainer + " on node " + node + ". This attempt currently has " + reservedContainers .Count + " reserved containers at priority " + priority + "; currentReservation " + currentReservation.GetMemory()); } return(rmContainer); } }
public static bool IsBlacklisted(SchedulerApplicationAttempt application, SchedulerNode node, Log Log) { if (application.IsBlacklisted(node.GetNodeName())) { if (Log.IsDebugEnabled()) { Log.Debug("Skipping 'host' " + node.GetNodeName() + " for " + application.GetApplicationId () + " since it has been blacklisted"); } return(true); } if (application.IsBlacklisted(node.GetRackName())) { if (Log.IsDebugEnabled()) { Log.Debug("Skipping 'rack' " + node.GetRackName() + " for " + application.GetApplicationId () + " since it has been blacklisted"); } return(true); } return(false); }
protected internal virtual void UpdateMaximumAllocation(SchedulerNode node, bool add) { Org.Apache.Hadoop.Yarn.Api.Records.Resource totalResource = node.GetTotalResource (); maxAllocWriteLock.Lock(); try { if (add) { // added node int nodeMemory = totalResource.GetMemory(); if (nodeMemory > maxNodeMemory) { maxNodeMemory = nodeMemory; maximumAllocation.SetMemory(Math.Min(configuredMaximumAllocation.GetMemory(), maxNodeMemory )); } int nodeVCores = totalResource.GetVirtualCores(); if (nodeVCores > maxNodeVCores) { maxNodeVCores = nodeVCores; maximumAllocation.SetVirtualCores(Math.Min(configuredMaximumAllocation.GetVirtualCores (), maxNodeVCores)); } } else { // removed node if (maxNodeMemory == totalResource.GetMemory()) { maxNodeMemory = -1; } if (maxNodeVCores == totalResource.GetVirtualCores()) { maxNodeVCores = -1; } // We only have to iterate through the nodes if the current max memory // or vcores was equal to the removed node's if (maxNodeMemory == -1 || maxNodeVCores == -1) { foreach (KeyValuePair <NodeId, N> nodeEntry in nodes) { int nodeMemory = nodeEntry.Value.GetTotalResource().GetMemory(); if (nodeMemory > maxNodeMemory) { maxNodeMemory = nodeMemory; } int nodeVCores = nodeEntry.Value.GetTotalResource().GetVirtualCores(); if (nodeVCores > maxNodeVCores) { maxNodeVCores = nodeVCores; } } if (maxNodeMemory == -1) { // no nodes maximumAllocation.SetMemory(configuredMaximumAllocation.GetMemory()); } else { maximumAllocation.SetMemory(Math.Min(configuredMaximumAllocation.GetMemory(), maxNodeMemory )); } if (maxNodeVCores == -1) { // no nodes maximumAllocation.SetVirtualCores(configuredMaximumAllocation.GetVirtualCores()); } else { maximumAllocation.SetVirtualCores(Math.Min(configuredMaximumAllocation.GetVirtualCores (), maxNodeVCores)); } } } } finally { maxAllocWriteLock.Unlock(); } }
public SchedulerNodeReport(SchedulerNode node) { this.used = node.GetUsedResource(); this.avail = node.GetAvailableResource(); this.num = node.GetNumContainers(); }