public virtual void TestUpdateMaxAllocationUsesTotal()
{
int configuredMaxVCores = 20;
int configuredMaxMemory = 10 * 1024;
Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(configuredMaxMemory, configuredMaxVCores);
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
);
Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(0, 0);
Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 5);
Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 10);
SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo"
, 8080));
Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource
);
Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1);
SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar"
, 8081));
Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource
);
Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
scheduler.nodes = new Dictionary <NodeId, SchedulerNode>();
scheduler.nodes[mockNode1.GetNodeID()] = mockNode1;
scheduler.UpdateMaximumAllocation(mockNode1, true);
VerifyMaximumResourceCapability(fullResource1, scheduler);
scheduler.nodes[mockNode2.GetNodeID()] = mockNode2;
scheduler.UpdateMaximumAllocation(mockNode2, true);
VerifyMaximumResourceCapability(fullResource2, scheduler);
Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID());
scheduler.UpdateMaximumAllocation(mockNode2, false);
VerifyMaximumResourceCapability(fullResource1, scheduler);
Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID());
scheduler.UpdateMaximumAllocation(mockNode1, false);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
}
finally
{
rm.Stop();
}
}
public virtual void TestMove()
{
string user = "******";
Queue parentQueue = CreateQueue("parent", null);
Queue oldQueue = CreateQueue("old", parentQueue);
Queue newQueue = CreateQueue("new", parentQueue);
QueueMetrics parentMetrics = parentQueue.GetMetrics();
QueueMetrics oldMetrics = oldQueue.GetMetrics();
QueueMetrics newMetrics = newQueue.GetMetrics();
ApplicationAttemptId appAttId = CreateAppAttemptId(0, 0);
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetEpoch()).ThenReturn(3L);
SchedulerApplicationAttempt app = new SchedulerApplicationAttempt(appAttId, user,
oldQueue, oldQueue.GetActiveUsersManager(), rmContext);
oldMetrics.SubmitApp(user);
// confirm that containerId is calculated based on epoch.
NUnit.Framework.Assert.AreEqual(unchecked ((long)(0x30000000001L)), app.GetNewContainerId
());
// Resource request
Resource requestedResource = Resource.NewInstance(1536, 2);
Priority requestedPriority = Priority.NewInstance(2);
ResourceRequest request = ResourceRequest.NewInstance(requestedPriority, ResourceRequest
.Any, requestedResource, 3);
app.UpdateResourceRequests(Arrays.AsList(request));
// Allocated container
RMContainer container1 = CreateRMContainer(appAttId, 1, requestedResource);
app.liveContainers[container1.GetContainerId()] = container1;
SchedulerNode node = CreateNode();
app.appSchedulingInfo.Allocate(NodeType.OffSwitch, node, requestedPriority, request
, container1.GetContainer());
// Reserved container
Priority prio1 = Priority.NewInstance(1);
Resource reservedResource = Resource.NewInstance(2048, 3);
RMContainer container2 = CreateReservedRMContainer(appAttId, 1, reservedResource,
node.GetNodeID(), prio1);
IDictionary <NodeId, RMContainer> reservations = new Dictionary <NodeId, RMContainer
>();
reservations[node.GetNodeID()] = container2;
app.reservedContainers[prio1] = reservations;
oldMetrics.ReserveResource(user, reservedResource);
CheckQueueMetrics(oldMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
CheckQueueMetrics(newMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
app.Move(newQueue);
CheckQueueMetrics(oldMetrics, 0, 0, 0, 0, 0, 0, 0, 0);
CheckQueueMetrics(newMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
CheckQueueMetrics(parentMetrics, 1, 1, 1536, 2, 2048, 3, 3072, 4);
}
private SchedulerNode CreateNode()
{
SchedulerNode node = Org.Mockito.Mockito.Mock <SchedulerNode>();
Org.Mockito.Mockito.When(node.GetNodeName()).ThenReturn("somehost");
Org.Mockito.Mockito.When(node.GetRackName()).ThenReturn("somerack");
Org.Mockito.Mockito.When(node.GetNodeID()).ThenReturn(nodeId);
return(node);
}
protected internal virtual void ContainerLaunchedOnNode(ContainerId containerId,
SchedulerNode node)
{
lock (this)
{
// Get the application for the finished container
SchedulerApplicationAttempt application = GetCurrentAttemptForContainer(containerId
);
if (application == null)
{
Log.Info("Unknown application " + containerId.GetApplicationAttemptId().GetApplicationId
() + " launched container " + containerId + " on node: " + node);
this.rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeCleanContainerEvent
(node.GetNodeID(), containerId));
return;
}
application.ContainerLaunchedOnNode(containerId, node.GetNodeID());
}
}
/// <summary>
/// Has the application reserved the given <code>node</code> at the
/// given <code>priority</code>?
/// </summary>
/// <param name="node">node to be checked</param>
/// <param name="priority">priority of reserved container</param>
/// <returns>true is reserved, false if not</returns>
public virtual bool IsReserved(SchedulerNode node, Priority priority)
{
lock (this)
{
IDictionary <NodeId, RMContainer> reservedContainers = this.reservedContainers[priority
];
if (reservedContainers != null)
{
return(reservedContainers.Contains(node.GetNodeID()));
}
return(false);
}
}
public virtual RMContainer Reserve(SchedulerNode node, Priority priority, RMContainer
rmContainer, Container container)
{
lock (this)
{
// Create RMContainer if necessary
if (rmContainer == null)
{
rmContainer = new RMContainerImpl(container, GetApplicationAttemptId(), node.GetNodeID
(), appSchedulingInfo.GetUser(), rmContext);
Resources.AddTo(currentReservation, container.GetResource());
// Reset the re-reservation count
ResetReReservations(priority);
}
else
{
// Note down the re-reservation
AddReReservation(priority);
}
rmContainer.Handle(new RMContainerReservedEvent(container.GetId(), container.GetResource
(), node.GetNodeID(), priority));
IDictionary <NodeId, RMContainer> reservedContainers = this.reservedContainers[priority
];
if (reservedContainers == null)
{
reservedContainers = new Dictionary <NodeId, RMContainer>();
this.reservedContainers[priority] = reservedContainers;
}
reservedContainers[node.GetNodeID()] = rmContainer;
if (Log.IsDebugEnabled())
{
Log.Debug("Application attempt " + GetApplicationAttemptId() + " reserved container "
+ rmContainer + " on node " + node + ". This attempt currently has " + reservedContainers
.Count + " reserved containers at priority " + priority + "; currentReservation "
+ currentReservation.GetMemory());
}
return(rmContainer);
}
}