public virtual void TestUpdateMaxAllocationUsesTotal()
{
int configuredMaxVCores = 20;
int configuredMaxMemory = 10 * 1024;
Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(configuredMaxMemory, configuredMaxVCores);
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
);
Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(0, 0);
Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 5);
Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 10);
SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo"
, 8080));
Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource
);
Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1);
SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar"
, 8081));
Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource
);
Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
scheduler.nodes = new Dictionary <NodeId, SchedulerNode>();
scheduler.nodes[mockNode1.GetNodeID()] = mockNode1;
scheduler.UpdateMaximumAllocation(mockNode1, true);
VerifyMaximumResourceCapability(fullResource1, scheduler);
scheduler.nodes[mockNode2.GetNodeID()] = mockNode2;
scheduler.UpdateMaximumAllocation(mockNode2, true);
VerifyMaximumResourceCapability(fullResource2, scheduler);
Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID());
scheduler.UpdateMaximumAllocation(mockNode2, false);
VerifyMaximumResourceCapability(fullResource1, scheduler);
Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID());
scheduler.UpdateMaximumAllocation(mockNode1, false);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
}
finally
{
rm.Stop();
}
}
/// <summary>Process resource update on a node.</summary>
public virtual void UpdateNodeResource(RMNode nm, ResourceOption resourceOption)
{
lock (this)
{
SchedulerNode node = GetSchedulerNode(nm.GetNodeID());
Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = resourceOption.GetResource
();
Org.Apache.Hadoop.Yarn.Api.Records.Resource oldResource = node.GetTotalResource();
if (!oldResource.Equals(newResource))
{
// Log resource change
Log.Info("Update resource on node: " + node.GetNodeName() + " from: " + oldResource
+ ", to: " + newResource);
Sharpen.Collections.Remove(nodes, nm.GetNodeID());
UpdateMaximumAllocation(node, false);
// update resource to node
node.SetTotalResource(newResource);
nodes[nm.GetNodeID()] = (N)node;
UpdateMaximumAllocation(node, true);
// update resource to clusterResource
Resources.SubtractFrom(clusterResource, oldResource);
Resources.AddTo(clusterResource, newResource);
}
else
{
// Log resource change
Log.Warn("Update resource on node: " + node.GetNodeName() + " with the same resource: "
+ newResource);
}
}
}
private SchedulerNode CreateNode()
{
SchedulerNode node = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(node.GetNodeName()).ThenReturn("somehost");
Org.Mockito.Mockito.When(node.GetRackName()).ThenReturn("somerack");
Org.Mockito.Mockito.When(node.GetNodeID()).ThenReturn(nodeId);
return(node);
}
public virtual void TestMove()
{
string user = "******";
Queue parentQueue = CreateQueue("parent", null);
Queue oldQueue = CreateQueue("old", parentQueue);
Queue newQueue = CreateQueue("new", parentQueue);
QueueMetrics parentMetrics = parentQueue.GetMetrics();
QueueMetrics oldMetrics = oldQueue.GetMetrics();
QueueMetrics newMetrics = newQueue.GetMetrics();
ApplicationAttemptId appAttId = CreateAppAttemptId(0, 0);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetEpoch()).ThenReturn(3L);
SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user,
oldQueue, oldQueue.GetActiveUsersManager(), rmContext);
oldMetrics.SubmitApp(user);
// confirm that containerId is calculated based on epoch.
NUnit.Framework.Assert.AreEqual(unchecked ((long)(0x30000000001L)), app.GetNewContainerId
());
// Resource request
Resource requestedResource = Resource.NewInstance(1536, 2);
Priority requestedPriority = Priority.NewInstance(2);
ResourceRequest request = ResourceRequest.NewInstance(requestedPriority, ResourceRequest
.Any, requestedResource, 3);
app.UpdateResourceRequests(Arrays.AsList(request));
// Allocated container
RMContainer container1 = CreateRMContainer(appAttId, 1, requestedResource);
app.liveContainers[container1.GetContainerId()] = container1;
SchedulerNode node = CreateNode();
app.appSchedulingInfo.Allocate(NodeType.OffSwitch, node, requestedPriority, request
, container1.GetContainer());
// Reserved container
Priority prio1 = Priority.NewInstance(1);
Resource reservedResource = Resource.NewInstance(2048, 3);
RMContainer container2 = CreateReservedRMContainer(appAttId, 1, reservedResource,
node.GetNodeID(), prio1);
IDictionary <NodeId, RMContainer> reservations = new Dictionary <NodeId, RMContainer
>();
reservations[node.GetNodeID()] = container2;
app.reservedContainers[prio1] = reservations;
oldMetrics.ReserveResource(user, reservedResource);
CheckQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
CheckQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
app.Move(newQueue);
CheckQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
CheckQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
}
/// <summary>
/// The
/// <see cref="ResourceScheduler"/>
/// is allocating data-local resources to the
/// application.
/// </summary>
/// <param name="allocatedContainers">resources allocated to the application</param>
private void AllocateOffSwitch(SchedulerNode node, Priority priority, ResourceRequest
offSwitchRequest, Container container, IList <ResourceRequest> resourceRequests)
{
lock (this)
{
// Update future requirements
DecrementOutstanding(offSwitchRequest);
// Update cloned OffRack requests for recovery
resourceRequests.AddItem(CloneResourceRequest(offSwitchRequest));
}
}
/// <summary>
/// Has the application reserved the given <code>node</code> at the
/// given <code>priority</code>?
/// </summary>
/// <param name="node">node to be checked</param>
/// <param name="priority">priority of reserved container</param>
/// <returns>true is reserved, false if not</returns>
public virtual bool IsReserved(SchedulerNode node, Priority priority)
{
lock (this)
{
IDictionary <NodeId, RMContainer> reservedContainers = this.reservedContainers[priority
];
if (reservedContainers != null)
{
return(reservedContainers.Contains(node.GetNodeID()));
}
return(false);
}
}
/// <summary>
/// The
/// <see cref="ResourceScheduler"/>
/// is allocating data-local resources to the
/// application.
/// </summary>
/// <param name="allocatedContainers">resources allocated to the application</param>
private void AllocateRackLocal(SchedulerNode node, Priority priority, ResourceRequest
rackLocalRequest, Container container, IList <ResourceRequest> resourceRequests)
{
lock (this)
{
// Update future requirements
DecResourceRequest(node.GetRackName(), priority, rackLocalRequest);
ResourceRequest offRackRequest = requests[priority][ResourceRequest.Any];
DecrementOutstanding(offRackRequest);
// Update cloned RackLocal and OffRack requests for recovery
resourceRequests.AddItem(CloneResourceRequest(rackLocalRequest));
resourceRequests.AddItem(CloneResourceRequest(offRackRequest));
}
}
protected internal virtual void ContainerLaunchedOnNode(ContainerId containerId,
SchedulerNode node)
{
lock (this)
{
// Get the application for the finished container
SchedulerApplicationAttempt application = GetCurrentAttemptForContainer(containerId
);
if (application == null)
{
Log.Info("Unknown application " + containerId.GetApplicationAttemptId().GetApplicationId
() + " launched container " + containerId + " on node: " + node);
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeCleanContainerEvent
(node.GetNodeID(), containerId));
return;
}
application.ContainerLaunchedOnNode(containerId, node.GetNodeID());
}
}
/// <summary>
/// Resources have been allocated to this application by the resource
/// scheduler.
/// </summary>
/// <remarks>
/// Resources have been allocated to this application by the resource
/// scheduler. Track them.
/// </remarks>
/// <param name="type">the type of the node</param>
/// <param name="node">the nodeinfo of the node</param>
/// <param name="priority">the priority of the request.</param>
/// <param name="request">the request</param>
/// <param name="container">the containers allocated.</param>
public virtual IList <ResourceRequest> Allocate(NodeType type, SchedulerNode node,
Priority priority, ResourceRequest request, Container container)
{
lock (this)
{
IList <ResourceRequest> resourceRequests = new AList <ResourceRequest>();
if (type == NodeType.NodeLocal)
{
AllocateNodeLocal(node, priority, request, container, resourceRequests);
}
else
{
if (type == NodeType.RackLocal)
{
AllocateRackLocal(node, priority, request, container, resourceRequests);
}
else
{
AllocateOffSwitch(node, priority, request, container, resourceRequests);
}
}
QueueMetrics metrics = queue.GetMetrics();
if (pending)
{
// once an allocation is done we assume the application is
// running from scheduler's POV.
pending = false;
metrics.RunAppAttempt(applicationId, user);
}
if (Log.IsDebugEnabled())
{
Log.Debug("allocate: applicationId=" + applicationId + " container=" + container.
GetId() + " host=" + container.GetNodeId().ToString() + " user="******" resource="
+ request.GetCapability());
}
metrics.AllocateResources(user, 1, request.GetCapability(), true);
return(resourceRequests);
}
}
public virtual RMContainer Reserve(SchedulerNode node, Priority priority, RMContainer
rmContainer, Container container)
{
lock (this)
{
// Create RMContainer if necessary
if (rmContainer == null)
{
rmContainer = new RMContainerImpl(container, GetApplicationAttemptId(), node.GetNodeID
(), appSchedulingInfo.GetUser(), rmContext);
Resources.AddTo(currentReservation, container.GetResource());
// Reset the re-reservation count
ResetReReservations(priority);
}
else
{
// Note down the re-reservation
AddReReservation(priority);
}
rmContainer.Handle(new RMContainerReservedEvent(container.GetId(), container.GetResource
(), node.GetNodeID(), priority));
IDictionary <NodeId, RMContainer> reservedContainers = this.reservedContainers[priority
];
if (reservedContainers == null)
{
reservedContainers = new Dictionary <NodeId, RMContainer>();
this.reservedContainers[priority] = reservedContainers;
}
reservedContainers[node.GetNodeID()] = rmContainer;
if (Log.IsDebugEnabled())
{
Log.Debug("Application attempt " + GetApplicationAttemptId() + " reserved container "
+ rmContainer + " on node " + node + ". This attempt currently has " + reservedContainers
.Count + " reserved containers at priority " + priority + "; currentReservation "
+ currentReservation.GetMemory());
}
return(rmContainer);
}
}
public static bool IsBlacklisted(SchedulerApplicationAttempt application, SchedulerNode
node, Log Log)
{
if (application.IsBlacklisted(node.GetNodeName()))
{
if (Log.IsDebugEnabled())
{
Log.Debug("Skipping 'host' " + node.GetNodeName() + " for " + application.GetApplicationId
() + " since it has been blacklisted");
}
return(true);
}
if (application.IsBlacklisted(node.GetRackName()))
{
if (Log.IsDebugEnabled())
{
Log.Debug("Skipping 'rack' " + node.GetRackName() + " for " + application.GetApplicationId
() + " since it has been blacklisted");
}
return(true);
}
return(false);
}
protected internal virtual void UpdateMaximumAllocation(SchedulerNode node, bool
add)
{
Org.Apache.Hadoop.Yarn.Api.Records.Resource totalResource = node.GetTotalResource
();
maxAllocWriteLock.Lock();
try
{
if (add)
{
// added node
int nodeMemory = totalResource.GetMemory();
if (nodeMemory > maxNodeMemory)
{
maxNodeMemory = nodeMemory;
maximumAllocation.SetMemory(Math.Min(configuredMaximumAllocation.GetMemory(), maxNodeMemory
));
}
int nodeVCores = totalResource.GetVirtualCores();
if (nodeVCores > maxNodeVCores)
{
maxNodeVCores = nodeVCores;
maximumAllocation.SetVirtualCores(Math.Min(configuredMaximumAllocation.GetVirtualCores
(), maxNodeVCores));
}
}
else
{
// removed node
if (maxNodeMemory == totalResource.GetMemory())
{
maxNodeMemory = -1;
}
if (maxNodeVCores == totalResource.GetVirtualCores())
{
maxNodeVCores = -1;
}
// We only have to iterate through the nodes if the current max memory
// or vcores was equal to the removed node's
if (maxNodeMemory == -1 || maxNodeVCores == -1)
{
foreach (KeyValuePair <NodeId, N> nodeEntry in nodes)
{
int nodeMemory = nodeEntry.Value.GetTotalResource().GetMemory();
if (nodeMemory > maxNodeMemory)
{
maxNodeMemory = nodeMemory;
}
int nodeVCores = nodeEntry.Value.GetTotalResource().GetVirtualCores();
if (nodeVCores > maxNodeVCores)
{
maxNodeVCores = nodeVCores;
}
}
if (maxNodeMemory == -1)
{
// no nodes
maximumAllocation.SetMemory(configuredMaximumAllocation.GetMemory());
}
else
{
maximumAllocation.SetMemory(Math.Min(configuredMaximumAllocation.GetMemory(), maxNodeMemory
));
}
if (maxNodeVCores == -1)
{
// no nodes
maximumAllocation.SetVirtualCores(configuredMaximumAllocation.GetVirtualCores());
}
else
{
maximumAllocation.SetVirtualCores(Math.Min(configuredMaximumAllocation.GetVirtualCores
(), maxNodeVCores));
}
}
}
}
finally
{
maxAllocWriteLock.Unlock();
}
}
public SchedulerNodeReport(SchedulerNode node)
{
this.used = node.GetUsedResource();
this.avail = node.GetAvailableResource();
this.num = node.GetNumContainers();
}
