private void ApplyRequestLimits() { IEnumerator <ResourceRequest> iter = requestLimits.Values.GetEnumerator(); while (iter.HasNext()) { ResourceRequest reqLimit = iter.Next(); int limit = reqLimit.GetNumContainers(); IDictionary <string, IDictionary <Resource, ResourceRequest> > remoteRequests = remoteRequestsTable [reqLimit.GetPriority()]; IDictionary <Resource, ResourceRequest> reqMap = (remoteRequests != null) ? remoteRequests [ResourceRequest.Any] : null; ResourceRequest req = (reqMap != null) ? reqMap[reqLimit.GetCapability()] : null; if (req == null) { continue; } // update an existing ask or send a new one if updating if (ask.Remove(req) || requestLimitsToUpdate.Contains(req)) { ResourceRequest newReq = req.GetNumContainers() > limit ? reqLimit : req; ask.AddItem(newReq); Log.Info("Applying ask limit of " + newReq.GetNumContainers() + " for priority:" + reqLimit.GetPriority() + " and capability:" + reqLimit.GetCapability()); } if (limit == int.MaxValue) { iter.Remove(); } } requestLimitsToUpdate.Clear(); }
/// <summary> /// Utility method to validate a resource request, by insuring that the /// requested memory/vcore is non-negative and not greater than max /// </summary> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.InvalidResourceRequestException /// ">when there is invalid request</exception> private static void ValidateResourceRequest(ResourceRequest resReq, Org.Apache.Hadoop.Yarn.Api.Records.Resource maximumResource, QueueInfo queueInfo, RMContext rmContext) { if (resReq.GetCapability().GetMemory() < 0 || resReq.GetCapability().GetMemory() > maximumResource.GetMemory()) { throw new InvalidResourceRequestException("Invalid resource request" + ", requested memory < 0" + ", or requested memory > max configured" + ", requestedMemory=" + resReq.GetCapability ().GetMemory() + ", maxMemory=" + maximumResource.GetMemory()); } if (resReq.GetCapability().GetVirtualCores() < 0 || resReq.GetCapability().GetVirtualCores () > maximumResource.GetVirtualCores()) { throw new InvalidResourceRequestException("Invalid resource request" + ", requested virtual cores < 0" + ", or requested virtual cores > max configured" + ", requestedVirtualCores=" + resReq.GetCapability().GetVirtualCores() + ", maxVirtualCores=" + maximumResource .GetVirtualCores()); } string labelExp = resReq.GetNodeLabelExpression(); // we don't allow specify label expression other than resourceName=ANY now if (!ResourceRequest.Any.Equals(resReq.GetResourceName()) && labelExp != null && !labelExp.Trim().IsEmpty()) { throw new InvalidResourceRequestException("Invailid resource request, queue=" + queueInfo .GetQueueName() + " specified node label expression in a " + "resource request has resource name = " + resReq.GetResourceName()); } // we don't allow specify label expression with more than one node labels now if (labelExp != null && labelExp.Contains("&&")) { throw new InvalidResourceRequestException("Invailid resource request, queue=" + queueInfo .GetQueueName() + " specified more than one node label " + "in a node label expression, node label expression = " + labelExp); } if (labelExp != null && !labelExp.Trim().IsEmpty() && queueInfo != null) { if (!CheckQueueLabelExpression(queueInfo.GetAccessibleNodeLabels(), labelExp, rmContext )) { throw new InvalidResourceRequestException("Invalid resource request" + ", queue=" + queueInfo.GetQueueName() + " doesn't have permission to access all labels " + "in resource request. labelExpression of resource request=" + labelExp + ". Queue labels=" + (queueInfo.GetAccessibleNodeLabels() == null ? string.Empty : StringUtils.Join (queueInfo.GetAccessibleNodeLabels().GetEnumerator(), ','))); } } }
/// <summary> /// Assign a container to this node to facilitate /// <paramref name="request"/> /// . If node does /// not have enough memory, create a reservation. This is called once we are /// sure the particular request should be facilitated by this node. /// </summary> /// <param name="node">The node to try placing the container on.</param> /// <param name="request">The ResourceRequest we're trying to satisfy.</param> /// <param name="type">The locality of the assignment.</param> /// <param name="reserved">Whether there's already a container reserved for this app on the node. /// </param> /// <returns> /// If an assignment was made, returns the resources allocated to the /// container. If a reservation was made, returns /// FairScheduler.CONTAINER_RESERVED. If no assignment or reservation was /// made, returns an empty resource. /// </returns> private Org.Apache.Hadoop.Yarn.Api.Records.Resource AssignContainer(FSSchedulerNode node, ResourceRequest request, NodeType type, bool reserved) { // How much does this request need? Org.Apache.Hadoop.Yarn.Api.Records.Resource capability = request.GetCapability(); // How much does the node have? Org.Apache.Hadoop.Yarn.Api.Records.Resource available = node.GetAvailableResource (); Container container = null; if (reserved) { container = node.GetReservedContainer().GetContainer(); } else { container = CreateContainer(node, capability, request.GetPriority()); } // Can we allocate a container on this node? if (Resources.FitsIn(capability, available)) { // Inform the application of the new container for this request RMContainer allocatedContainer = Allocate(type, node, request.GetPriority(), request , container); if (allocatedContainer == null) { // Did the application need this resource? if (reserved) { Unreserve(request.GetPriority(), node); } return(Resources.None()); } // If we had previously made a reservation, delete it if (reserved) { Unreserve(request.GetPriority(), node); } // Inform the node node.AllocateContainer(allocatedContainer); // If this container is used to run AM, update the leaf queue's AM usage if (GetLiveContainers().Count == 1 && !GetUnmanagedAM()) { ((FSLeafQueue)GetQueue()).AddAMResourceUsage(container.GetResource()); SetAmRunning(true); } return(container.GetResource()); } else { if (!FairScheduler.FitsInMaxShare(((FSLeafQueue)GetQueue()), capability)) { return(Resources.None()); } // The desired container won't fit here, so reserve Reserve(request.GetPriority(), node, container, reserved); return(FairScheduler.ContainerReserved); } }
public virtual Org.Apache.Hadoop.Yarn.Api.Records.Resource GetResource(Priority priority ) { lock (this) { ResourceRequest request = GetResourceRequest(priority, ResourceRequest.Any); return((request == null) ? null : request.GetCapability()); } }
public static ResourceRequest NewResourceRequest(ResourceRequest r) { ResourceRequest request = recordFactory.NewRecordInstance <ResourceRequest>(); request.SetPriority(r.GetPriority()); request.SetResourceName(r.GetResourceName()); request.SetCapability(r.GetCapability()); request.SetNumContainers(r.GetNumContainers()); return(request); }
/// <summary> /// Resources have been allocated to this application by the resource /// scheduler. /// </summary> /// <remarks> /// Resources have been allocated to this application by the resource /// scheduler. Track them. /// </remarks> /// <param name="type">the type of the node</param> /// <param name="node">the nodeinfo of the node</param> /// <param name="priority">the priority of the request.</param> /// <param name="request">the request</param> /// <param name="container">the containers allocated.</param> public virtual IList <ResourceRequest> Allocate(NodeType type, SchedulerNode node, Priority priority, ResourceRequest request, Container container) { lock (this) { IList <ResourceRequest> resourceRequests = new AList <ResourceRequest>(); if (type == NodeType.NodeLocal) { AllocateNodeLocal(node, priority, request, container, resourceRequests); } else { if (type == NodeType.RackLocal) { AllocateRackLocal(node, priority, request, container, resourceRequests); } else { AllocateOffSwitch(node, priority, request, container, resourceRequests); } } QueueMetrics metrics = queue.GetMetrics(); if (pending) { // once an allocation is done we assume the application is // running from scheduler's POV. pending = false; metrics.RunAppAttempt(applicationId, user); } if (Log.IsDebugEnabled()) { Log.Debug("allocate: applicationId=" + applicationId + " container=" + container. GetId() + " host=" + container.GetNodeId().ToString() + " user="******" resource=" + request.GetCapability()); } metrics.AllocateResources(user, 1, request.GetCapability(), true); return(resourceRequests); } }
private int AssignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application, Priority priority, int assignableContainers, ResourceRequest request, NodeType type ) { Log.Debug("assignContainers:" + " node=" + node.GetRMNode().GetNodeAddress() + " application=" + application.GetApplicationId().GetId() + " priority=" + priority.GetPriority( ) + " assignableContainers=" + assignableContainers + " request=" + request + " type=" + type); Org.Apache.Hadoop.Yarn.Api.Records.Resource capability = request.GetCapability(); int availableContainers = node.GetAvailableResource().GetMemory() / capability.GetMemory (); // TODO: A buggy // application // with this // zero would // crash the // scheduler. int assignedContainers = Math.Min(assignableContainers, availableContainers); if (assignedContainers > 0) { for (int i = 0; i < assignedContainers; ++i) { NodeId nodeId = node.GetRMNode().GetNodeID(); ContainerId containerId = BuilderUtils.NewContainerId(application.GetApplicationAttemptId (), application.GetNewContainerId()); // Create the container Container container = BuilderUtils.NewContainer(containerId, nodeId, node.GetRMNode ().GetHttpAddress(), capability, priority, null); // Allocate! // Inform the application RMContainer rmContainer = application.Allocate(type, node, priority, request, container ); // Inform the node node.AllocateContainer(rmContainer); // Update usage for this container IncreaseUsedResources(rmContainer); } } return(assignedContainers); }
/// <summary> /// Whether this app has containers requests that could be satisfied on the /// given node, if the node had full space. /// </summary> public virtual bool HasContainerForNode(Priority prio, FSSchedulerNode node) { ResourceRequest anyRequest = GetResourceRequest(prio, ResourceRequest.Any); ResourceRequest rackRequest = GetResourceRequest(prio, node.GetRackName()); ResourceRequest nodeRequest = GetResourceRequest(prio, node.GetNodeName()); return(anyRequest != null && anyRequest.GetNumContainers() > 0 && (anyRequest.GetRelaxLocality () || (rackRequest != null && rackRequest.GetNumContainers() > 0)) && (rackRequest == null || rackRequest.GetRelaxLocality() || (nodeRequest != null && nodeRequest .GetNumContainers() > 0)) && Resources.LessThanOrEqual(ResourceCalculator, null, anyRequest.GetCapability(), node.GetRMNode().GetTotalCapability())); }
public virtual ResourceRequest CloneResourceRequest(ResourceRequest request) { ResourceRequest newRequest = ResourceRequest.NewInstance(request.GetPriority(), request .GetResourceName(), request.GetCapability(), 1, request.GetRelaxLocality()); return(newRequest); }
public virtual void Stop(RMAppAttemptState rmAppAttemptFinalState) { lock (this) { // clear pending resources metrics for the application QueueMetrics metrics = queue.GetMetrics(); foreach (IDictionary <string, ResourceRequest> asks in requests.Values) { ResourceRequest request = asks[ResourceRequest.Any]; if (request != null) { metrics.DecrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); } } metrics.FinishAppAttempt(applicationId, pending, user); // Clear requests themselves ClearRequests(); } }
public virtual void Move(Queue newQueue) { lock (this) { QueueMetrics oldMetrics = queue.GetMetrics(); QueueMetrics newMetrics = newQueue.GetMetrics(); foreach (IDictionary <string, ResourceRequest> asks in requests.Values) { ResourceRequest request = asks[ResourceRequest.Any]; if (request != null) { oldMetrics.DecrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); newMetrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); } } oldMetrics.MoveAppFrom(this); newMetrics.MoveAppTo(this); activeUsersManager.DeactivateApplication(user, applicationId); activeUsersManager = newQueue.GetActiveUsersManager(); activeUsersManager.ActivateApplication(user, applicationId); this.queue = newQueue; this.queueName = newQueue.GetQueueName(); } }
/// <summary> /// The ApplicationMaster is updating resource requirements for the /// application, by asking for more resources and releasing resources acquired /// by the application. /// </summary> /// <param name="requests">resources to be acquired</param> /// <param name="recoverPreemptedRequest">recover Resource Request on preemption</param> public virtual void UpdateResourceRequests(IList <ResourceRequest> requests, bool recoverPreemptedRequest) { lock (this) { QueueMetrics metrics = queue.GetMetrics(); // Update resource requests foreach (ResourceRequest request in requests) { Priority priority = request.GetPriority(); string resourceName = request.GetResourceName(); bool updatePendingResources = false; ResourceRequest lastRequest = null; if (resourceName.Equals(ResourceRequest.Any)) { if (Log.IsDebugEnabled()) { Log.Debug("update:" + " application=" + applicationId + " request=" + request); } updatePendingResources = true; // Premature optimization? // Assumes that we won't see more than one priority request updated // in one call, reasonable assumption... however, it's totally safe // to activate same application more than once. // Thus we don't need another loop ala the one in decrementOutstanding() // which is needed during deactivate. if (request.GetNumContainers() > 0) { activeUsersManager.ActivateApplication(user, applicationId); } } IDictionary <string, ResourceRequest> asks = this.requests[priority]; if (asks == null) { asks = new ConcurrentHashMap <string, ResourceRequest>(); this.requests[priority] = asks; this.priorities.AddItem(priority); } lastRequest = asks[resourceName]; if (recoverPreemptedRequest && lastRequest != null) { // Increment the number of containers to 1, as it is recovering a // single container. request.SetNumContainers(lastRequest.GetNumContainers() + 1); } asks[resourceName] = request; if (updatePendingResources) { // Similarly, deactivate application? if (request.GetNumContainers() <= 0) { Log.Info("checking for deactivate of application :" + this.applicationId); CheckForDeactivation(); } int lastRequestContainers = lastRequest != null?lastRequest.GetNumContainers() : 0; Org.Apache.Hadoop.Yarn.Api.Records.Resource lastRequestCapability = lastRequest != null?lastRequest.GetCapability() : Resources.None(); metrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability ()); metrics.DecrPendingResources(user, lastRequestContainers, lastRequestCapability); } } } }
/// <summary> /// Utility method to normalize a resource request, by insuring that the /// requested memory is a multiple of minMemory and is not zero. /// </summary> public static void NormalizeRequest(ResourceRequest ask, ResourceCalculator resourceCalculator , Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource, Org.Apache.Hadoop.Yarn.Api.Records.Resource minimumResource, Org.Apache.Hadoop.Yarn.Api.Records.Resource maximumResource, Org.Apache.Hadoop.Yarn.Api.Records.Resource incrementResource) { Org.Apache.Hadoop.Yarn.Api.Records.Resource normalized = Resources.Normalize(resourceCalculator , ask.GetCapability(), minimumResource, maximumResource, incrementResource); ask.SetCapability(normalized); }
public virtual void TestHeadroom() { CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(); csConf.SetUserLimit(CapacitySchedulerConfiguration.Root + "." + A, 25); SetupQueueConfiguration(csConf); YarnConfiguration conf = new YarnConfiguration(); CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext >(); Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf); Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf); Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources .CreateResource(Gb)); Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources .CreateResource(16 * Gb)); Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler .applicationComparator); Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler .queueComparator); Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator ); Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext); // Say cluster has 100 nodes of 16G each Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (100 * 16 * Gb); Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource ); IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>(); CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues, TestUtils .spyHook); // Manipulate queue 'a' LeafQueue queue = TestLeafQueue.StubLeafQueue((LeafQueue)queues[A]); string host_0 = "host_0"; string rack_0 = "rack_0"; FiCaSchedulerNode node_0 = TestUtils.GetMockNode(host_0, rack_0, 0, 16 * Gb); string user_0 = "user_0"; string user_1 = "user_1"; RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null); RMContext rmContext = TestUtils.GetMockRMContext(); RMContext spyRMContext = Org.Mockito.Mockito.Spy(rmContext); ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap <ApplicationId, RMApp>()); RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>(); ResourceRequest amResourceRequest = Org.Mockito.Mockito.Mock <ResourceRequest>(); Org.Apache.Hadoop.Yarn.Api.Records.Resource amResource = Resources.CreateResource (0, 0); Org.Mockito.Mockito.When(amResourceRequest.GetCapability()).ThenReturn(amResource ); Org.Mockito.Mockito.When(rmApp.GetAMResourceRequest()).ThenReturn(amResourceRequest ); Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()]; Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps); Priority priority_1 = TestUtils.CreateMockPriority(1); // Submit first application with some resource-requests from user_0, // and check headroom ApplicationAttemptId appAttemptId_0_0 = TestUtils.GetMockApplicationAttemptId(0, 0); FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue, queue.GetActiveUsersManager(), spyRMContext); queue.SubmitApplicationAttempt(app_0_0, user_0); IList <ResourceRequest> app_0_0_requests = new AList <ResourceRequest>(); app_0_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 * Gb, 2, true, priority_1, recordFactory)); app_0_0.UpdateResourceRequests(app_0_0_requests); // Schedule to compute queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); Org.Apache.Hadoop.Yarn.Api.Records.Resource expectedHeadroom = Resources.CreateResource (10 * 16 * Gb, 1); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom()); // Submit second application from user_0, check headroom ApplicationAttemptId appAttemptId_0_1 = TestUtils.GetMockApplicationAttemptId(1, 0); FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue, queue.GetActiveUsersManager(), spyRMContext); queue.SubmitApplicationAttempt(app_0_1, user_0); IList <ResourceRequest> app_0_1_requests = new AList <ResourceRequest>(); app_0_1_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 * Gb, 2, true, priority_1, recordFactory)); app_0_1.UpdateResourceRequests(app_0_1_requests); // Schedule to compute queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); // Schedule to compute NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom()); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom()); // no change // Submit first application from user_1, check for new headroom ApplicationAttemptId appAttemptId_1_0 = TestUtils.GetMockApplicationAttemptId(2, 0); FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue, queue.GetActiveUsersManager(), spyRMContext); queue.SubmitApplicationAttempt(app_1_0, user_1); IList <ResourceRequest> app_1_0_requests = new AList <ResourceRequest>(); app_1_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 * Gb, 2, true, priority_1, recordFactory)); app_1_0.UpdateResourceRequests(app_1_0_requests); // Schedule to compute queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); // Schedule to compute expectedHeadroom = Resources.CreateResource(10 * 16 * Gb / 2, 1); // changes NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom()); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom()); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom()); // Now reduce cluster size and check for the smaller headroom clusterResource = Resources.CreateResource(90 * 16 * Gb); queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); // Schedule to compute expectedHeadroom = Resources.CreateResource(9 * 16 * Gb / 2, 1); // changes NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom()); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom()); NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom()); }