private void ApplyRequestLimits()
{
IEnumerator <ResourceRequest> iter = requestLimits.Values.GetEnumerator();
while (iter.HasNext())
{
ResourceRequest reqLimit = iter.Next();
int limit = reqLimit.GetNumContainers();
IDictionary <string, IDictionary <Resource, ResourceRequest> > remoteRequests = remoteRequestsTable
[reqLimit.GetPriority()];
IDictionary <Resource, ResourceRequest> reqMap = (remoteRequests != null) ? remoteRequests
[ResourceRequest.Any] : null;
ResourceRequest req = (reqMap != null) ? reqMap[reqLimit.GetCapability()] : null;
if (req == null)
{
continue;
}
// update an existing ask or send a new one if updating
if (ask.Remove(req) || requestLimitsToUpdate.Contains(req))
{
ResourceRequest newReq = req.GetNumContainers() > limit ? reqLimit : req;
ask.AddItem(newReq);
Log.Info("Applying ask limit of " + newReq.GetNumContainers() + " for priority:"
+ reqLimit.GetPriority() + " and capability:" + reqLimit.GetCapability());
}
if (limit == int.MaxValue)
{
iter.Remove();
}
}
requestLimitsToUpdate.Clear();
}
/// <summary>
/// Utility method to validate a resource request, by insuring that the
/// requested memory/vcore is non-negative and not greater than max
/// </summary>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.InvalidResourceRequestException
/// ">when there is invalid request</exception>
private static void ValidateResourceRequest(ResourceRequest resReq, Org.Apache.Hadoop.Yarn.Api.Records.Resource
maximumResource, QueueInfo queueInfo, RMContext rmContext)
{
if (resReq.GetCapability().GetMemory() < 0 || resReq.GetCapability().GetMemory()
> maximumResource.GetMemory())
{
throw new InvalidResourceRequestException("Invalid resource request" + ", requested memory < 0"
+ ", or requested memory > max configured" + ", requestedMemory=" + resReq.GetCapability
().GetMemory() + ", maxMemory=" + maximumResource.GetMemory());
}
if (resReq.GetCapability().GetVirtualCores() < 0 || resReq.GetCapability().GetVirtualCores
() > maximumResource.GetVirtualCores())
{
throw new InvalidResourceRequestException("Invalid resource request" + ", requested virtual cores < 0"
+ ", or requested virtual cores > max configured" + ", requestedVirtualCores="
+ resReq.GetCapability().GetVirtualCores() + ", maxVirtualCores=" + maximumResource
.GetVirtualCores());
}
string labelExp = resReq.GetNodeLabelExpression();
// we don't allow specify label expression other than resourceName=ANY now
if (!ResourceRequest.Any.Equals(resReq.GetResourceName()) && labelExp != null &&
!labelExp.Trim().IsEmpty())
{
throw new InvalidResourceRequestException("Invailid resource request, queue=" + queueInfo
.GetQueueName() + " specified node label expression in a " + "resource request has resource name = "
+ resReq.GetResourceName());
}
// we don't allow specify label expression with more than one node labels now
if (labelExp != null && labelExp.Contains("&&"))
{
throw new InvalidResourceRequestException("Invailid resource request, queue=" + queueInfo
.GetQueueName() + " specified more than one node label " + "in a node label expression, node label expression = "
+ labelExp);
}
if (labelExp != null && !labelExp.Trim().IsEmpty() && queueInfo != null)
{
if (!CheckQueueLabelExpression(queueInfo.GetAccessibleNodeLabels(), labelExp, rmContext
))
{
throw new InvalidResourceRequestException("Invalid resource request" + ", queue="
+ queueInfo.GetQueueName() + " doesn't have permission to access all labels " +
"in resource request. labelExpression of resource request=" + labelExp + ". Queue labels="
+ (queueInfo.GetAccessibleNodeLabels() == null ? string.Empty : StringUtils.Join
(queueInfo.GetAccessibleNodeLabels().GetEnumerator(), ',')));
}
}
}
/// <summary>
/// Assign a container to this node to facilitate
/// <paramref name="request"/>
/// . If node does
/// not have enough memory, create a reservation. This is called once we are
/// sure the particular request should be facilitated by this node.
/// </summary>
/// <param name="node">The node to try placing the container on.</param>
/// <param name="request">The ResourceRequest we're trying to satisfy.</param>
/// <param name="type">The locality of the assignment.</param>
/// <param name="reserved">Whether there's already a container reserved for this app on the node.
/// </param>
/// <returns>
/// If an assignment was made, returns the resources allocated to the
/// container. If a reservation was made, returns
/// FairScheduler.CONTAINER_RESERVED. If no assignment or reservation was
/// made, returns an empty resource.
/// </returns>
private Org.Apache.Hadoop.Yarn.Api.Records.Resource AssignContainer(FSSchedulerNode
node, ResourceRequest request, NodeType type, bool reserved)
{
// How much does this request need?
Org.Apache.Hadoop.Yarn.Api.Records.Resource capability = request.GetCapability();
// How much does the node have?
Org.Apache.Hadoop.Yarn.Api.Records.Resource available = node.GetAvailableResource
();
Container container = null;
if (reserved)
{
container = node.GetReservedContainer().GetContainer();
}
else
{
container = CreateContainer(node, capability, request.GetPriority());
}
// Can we allocate a container on this node?
if (Resources.FitsIn(capability, available))
{
// Inform the application of the new container for this request
RMContainer allocatedContainer = Allocate(type, node, request.GetPriority(), request
, container);
if (allocatedContainer == null)
{
// Did the application need this resource?
if (reserved)
{
Unreserve(request.GetPriority(), node);
}
return(Resources.None());
}
// If we had previously made a reservation, delete it
if (reserved)
{
Unreserve(request.GetPriority(), node);
}
// Inform the node
node.AllocateContainer(allocatedContainer);
// If this container is used to run AM, update the leaf queue's AM usage
if (GetLiveContainers().Count == 1 && !GetUnmanagedAM())
{
((FSLeafQueue)GetQueue()).AddAMResourceUsage(container.GetResource());
SetAmRunning(true);
}
return(container.GetResource());
}
else
{
if (!FairScheduler.FitsInMaxShare(((FSLeafQueue)GetQueue()), capability))
{
return(Resources.None());
}
// The desired container won't fit here, so reserve
Reserve(request.GetPriority(), node, container, reserved);
return(FairScheduler.ContainerReserved);
}
}
public virtual Org.Apache.Hadoop.Yarn.Api.Records.Resource GetResource(Priority priority
)
{
lock (this)
{
ResourceRequest request = GetResourceRequest(priority, ResourceRequest.Any);
return((request == null) ? null : request.GetCapability());
}
}
public static ResourceRequest NewResourceRequest(ResourceRequest r)
{
ResourceRequest request = recordFactory.NewRecordInstance <ResourceRequest>();
request.SetPriority(r.GetPriority());
request.SetResourceName(r.GetResourceName());
request.SetCapability(r.GetCapability());
request.SetNumContainers(r.GetNumContainers());
return(request);
}
/// <summary>
/// Resources have been allocated to this application by the resource
/// scheduler.
/// </summary>
/// <remarks>
/// Resources have been allocated to this application by the resource
/// scheduler. Track them.
/// </remarks>
/// <param name="type">the type of the node</param>
/// <param name="node">the nodeinfo of the node</param>
/// <param name="priority">the priority of the request.</param>
/// <param name="request">the request</param>
/// <param name="container">the containers allocated.</param>
public virtual IList <ResourceRequest> Allocate(NodeType type, SchedulerNode node,
Priority priority, ResourceRequest request, Container container)
{
lock (this)
{
IList <ResourceRequest> resourceRequests = new AList <ResourceRequest>();
if (type == NodeType.NodeLocal)
{
AllocateNodeLocal(node, priority, request, container, resourceRequests);
}
else
{
if (type == NodeType.RackLocal)
{
AllocateRackLocal(node, priority, request, container, resourceRequests);
}
else
{
AllocateOffSwitch(node, priority, request, container, resourceRequests);
}
}
QueueMetrics metrics = queue.GetMetrics();
if (pending)
{
// once an allocation is done we assume the application is
// running from scheduler's POV.
pending = false;
metrics.RunAppAttempt(applicationId, user);
}
if (Log.IsDebugEnabled())
{
Log.Debug("allocate: applicationId=" + applicationId + " container=" + container.
GetId() + " host=" + container.GetNodeId().ToString() + " user="******" resource="
+ request.GetCapability());
}
metrics.AllocateResources(user, 1, request.GetCapability(), true);
return(resourceRequests);
}
}
private int AssignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application,
Priority priority, int assignableContainers, ResourceRequest request, NodeType type
)
{
Log.Debug("assignContainers:" + " node=" + node.GetRMNode().GetNodeAddress() + " application="
+ application.GetApplicationId().GetId() + " priority=" + priority.GetPriority(
) + " assignableContainers=" + assignableContainers + " request=" + request + " type="
+ type);
Org.Apache.Hadoop.Yarn.Api.Records.Resource capability = request.GetCapability();
int availableContainers = node.GetAvailableResource().GetMemory() / capability.GetMemory
();
// TODO: A buggy
// application
// with this
// zero would
// crash the
// scheduler.
int assignedContainers = Math.Min(assignableContainers, availableContainers);
if (assignedContainers > 0)
{
for (int i = 0; i < assignedContainers; ++i)
{
NodeId nodeId = node.GetRMNode().GetNodeID();
ContainerId containerId = BuilderUtils.NewContainerId(application.GetApplicationAttemptId
(), application.GetNewContainerId());
// Create the container
Container container = BuilderUtils.NewContainer(containerId, nodeId, node.GetRMNode
().GetHttpAddress(), capability, priority, null);
// Allocate!
// Inform the application
RMContainer rmContainer = application.Allocate(type, node, priority, request, container
);
// Inform the node
node.AllocateContainer(rmContainer);
// Update usage for this container
IncreaseUsedResources(rmContainer);
}
}
return(assignedContainers);
}
/// <summary>
/// Whether this app has containers requests that could be satisfied on the
/// given node, if the node had full space.
/// </summary>
public virtual bool HasContainerForNode(Priority prio, FSSchedulerNode node)
{
ResourceRequest anyRequest = GetResourceRequest(prio, ResourceRequest.Any);
ResourceRequest rackRequest = GetResourceRequest(prio, node.GetRackName());
ResourceRequest nodeRequest = GetResourceRequest(prio, node.GetNodeName());
return(anyRequest != null && anyRequest.GetNumContainers() > 0 && (anyRequest.GetRelaxLocality
() || (rackRequest != null && rackRequest.GetNumContainers() > 0)) && (rackRequest
== null || rackRequest.GetRelaxLocality() || (nodeRequest != null && nodeRequest
.GetNumContainers() > 0)) && Resources.LessThanOrEqual(ResourceCalculator, null,
anyRequest.GetCapability(), node.GetRMNode().GetTotalCapability()));
}
public virtual ResourceRequest CloneResourceRequest(ResourceRequest request)
{
ResourceRequest newRequest = ResourceRequest.NewInstance(request.GetPriority(), request
.GetResourceName(), request.GetCapability(), 1, request.GetRelaxLocality());
return(newRequest);
}
public virtual void Stop(RMAppAttemptState rmAppAttemptFinalState)
{
lock (this)
{
// clear pending resources metrics for the application
QueueMetrics metrics = queue.GetMetrics();
foreach (IDictionary <string, ResourceRequest> asks in requests.Values)
{
ResourceRequest request = asks[ResourceRequest.Any];
if (request != null)
{
metrics.DecrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
}
}
metrics.FinishAppAttempt(applicationId, pending, user);
// Clear requests themselves
ClearRequests();
}
}
public virtual void Move(Queue newQueue)
{
lock (this)
{
QueueMetrics oldMetrics = queue.GetMetrics();
QueueMetrics newMetrics = newQueue.GetMetrics();
foreach (IDictionary <string, ResourceRequest> asks in requests.Values)
{
ResourceRequest request = asks[ResourceRequest.Any];
if (request != null)
{
oldMetrics.DecrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
newMetrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
}
}
oldMetrics.MoveAppFrom(this);
newMetrics.MoveAppTo(this);
activeUsersManager.DeactivateApplication(user, applicationId);
activeUsersManager = newQueue.GetActiveUsersManager();
activeUsersManager.ActivateApplication(user, applicationId);
this.queue = newQueue;
this.queueName = newQueue.GetQueueName();
}
}
/// <summary>
/// The ApplicationMaster is updating resource requirements for the
/// application, by asking for more resources and releasing resources acquired
/// by the application.
/// </summary>
/// <param name="requests">resources to be acquired</param>
/// <param name="recoverPreemptedRequest">recover Resource Request on preemption</param>
public virtual void UpdateResourceRequests(IList <ResourceRequest> requests, bool
recoverPreemptedRequest)
{
lock (this)
{
QueueMetrics metrics = queue.GetMetrics();
// Update resource requests
foreach (ResourceRequest request in requests)
{
Priority priority = request.GetPriority();
string resourceName = request.GetResourceName();
bool updatePendingResources = false;
ResourceRequest lastRequest = null;
if (resourceName.Equals(ResourceRequest.Any))
{
if (Log.IsDebugEnabled())
{
Log.Debug("update:" + " application=" + applicationId + " request=" + request);
}
updatePendingResources = true;
// Premature optimization?
// Assumes that we won't see more than one priority request updated
// in one call, reasonable assumption... however, it's totally safe
// to activate same application more than once.
// Thus we don't need another loop ala the one in decrementOutstanding()
// which is needed during deactivate.
if (request.GetNumContainers() > 0)
{
activeUsersManager.ActivateApplication(user, applicationId);
}
}
IDictionary <string, ResourceRequest> asks = this.requests[priority];
if (asks == null)
{
asks = new ConcurrentHashMap <string, ResourceRequest>();
this.requests[priority] = asks;
this.priorities.AddItem(priority);
}
lastRequest = asks[resourceName];
if (recoverPreemptedRequest && lastRequest != null)
{
// Increment the number of containers to 1, as it is recovering a
// single container.
request.SetNumContainers(lastRequest.GetNumContainers() + 1);
}
asks[resourceName] = request;
if (updatePendingResources)
{
// Similarly, deactivate application?
if (request.GetNumContainers() <= 0)
{
Log.Info("checking for deactivate of application :" + this.applicationId);
CheckForDeactivation();
}
int lastRequestContainers = lastRequest != null?lastRequest.GetNumContainers() :
0;
Org.Apache.Hadoop.Yarn.Api.Records.Resource lastRequestCapability = lastRequest !=
null?lastRequest.GetCapability() : Resources.None();
metrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
metrics.DecrPendingResources(user, lastRequestContainers, lastRequestCapability);
}
}
}
}
/// <summary>
/// Utility method to normalize a resource request, by insuring that the
/// requested memory is a multiple of minMemory and is not zero.
/// </summary>
public static void NormalizeRequest(ResourceRequest ask, ResourceCalculator resourceCalculator
, Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource, Org.Apache.Hadoop.Yarn.Api.Records.Resource
minimumResource, Org.Apache.Hadoop.Yarn.Api.Records.Resource maximumResource, Org.Apache.Hadoop.Yarn.Api.Records.Resource
incrementResource)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource normalized = Resources.Normalize(resourceCalculator
, ask.GetCapability(), minimumResource, maximumResource, incrementResource);
ask.SetCapability(normalized);
}
public virtual void TestHeadroom()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
csConf.SetUserLimit(CapacitySchedulerConfiguration.Root + "." + A, 25);
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext
>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(Gb));
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(16 * Gb));
Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler
.applicationComparator);
Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler
.queueComparator);
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator
);
Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext);
// Say cluster has 100 nodes of 16G each
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(100 * 16 * Gb);
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(clusterResource
);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues, queues, TestUtils
.spyHook);
// Manipulate queue 'a'
LeafQueue queue = TestLeafQueue.StubLeafQueue((LeafQueue)queues[A]);
string host_0 = "host_0";
string rack_0 = "rack_0";
FiCaSchedulerNode node_0 = TestUtils.GetMockNode(host_0, rack_0, 0, 16 * Gb);
string user_0 = "user_0";
string user_1 = "user_1";
RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(null);
RMContext rmContext = TestUtils.GetMockRMContext();
RMContext spyRMContext = Org.Mockito.Mockito.Spy(rmContext);
ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
ResourceRequest amResourceRequest = Org.Mockito.Mockito.Mock <ResourceRequest>();
Org.Apache.Hadoop.Yarn.Api.Records.Resource amResource = Resources.CreateResource
(0, 0);
Org.Mockito.Mockito.When(amResourceRequest.GetCapability()).ThenReturn(amResource
);
Org.Mockito.Mockito.When(rmApp.GetAMResourceRequest()).ThenReturn(amResourceRequest
);
Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
Priority priority_1 = TestUtils.CreateMockPriority(1);
// Submit first application with some resource-requests from user_0,
// and check headroom
ApplicationAttemptId appAttemptId_0_0 = TestUtils.GetMockApplicationAttemptId(0,
0);
FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_0_0, user_0);
IList <ResourceRequest> app_0_0_requests = new AList <ResourceRequest>();
app_0_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_0_0.UpdateResourceRequests(app_0_0_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
Org.Apache.Hadoop.Yarn.Api.Records.Resource expectedHeadroom = Resources.CreateResource
(10 * 16 * Gb, 1);
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
// Submit second application from user_0, check headroom
ApplicationAttemptId appAttemptId_0_1 = TestUtils.GetMockApplicationAttemptId(1,
0);
FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_0_1, user_0);
IList <ResourceRequest> app_0_1_requests = new AList <ResourceRequest>();
app_0_1_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_0_1.UpdateResourceRequests(app_0_1_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
// no change
// Submit first application from user_1, check for new headroom
ApplicationAttemptId appAttemptId_1_0 = TestUtils.GetMockApplicationAttemptId(2,
0);
FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue,
queue.GetActiveUsersManager(), spyRMContext);
queue.SubmitApplicationAttempt(app_1_0, user_1);
IList <ResourceRequest> app_1_0_requests = new AList <ResourceRequest>();
app_1_0_requests.AddItem(TestUtils.CreateResourceRequest(ResourceRequest.Any, 1 *
Gb, 2, true, priority_1, recordFactory));
app_1_0.UpdateResourceRequests(app_1_0_requests);
// Schedule to compute
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
expectedHeadroom = Resources.CreateResource(10 * 16 * Gb / 2, 1);
// changes
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
// Now reduce cluster size and check for the smaller headroom
clusterResource = Resources.CreateResource(90 * 16 * Gb);
queue.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
// Schedule to compute
expectedHeadroom = Resources.CreateResource(9 * 16 * Gb / 2, 1);
// changes
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_0.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_0_1.GetHeadroom());
NUnit.Framework.Assert.AreEqual(expectedHeadroom, app_1_0.GetHeadroom());
}
