private void AddResourceRequest(Priority priority, IDictionary <string, ResourceRequest
> requests, string resourceName, Org.Apache.Hadoop.Yarn.Api.Records.Resource capability
)
{
lock (this)
{
ResourceRequest request = requests[resourceName];
if (request == null)
{
request = BuilderUtils.NewResourceRequest(priority, resourceName, capability, 1);
requests[resourceName] = request;
}
else
{
request.SetNumContainers(request.GetNumContainers() + 1);
}
// Note this down for next interaction with ResourceManager
ask.Remove(request);
ask.AddItem(BuilderUtils.NewResourceRequest(request));
// clone to ensure the RM doesn't manipulate the same obj
if (Log.IsDebugEnabled())
{
Log.Debug("addResourceRequest: applicationId=" + applicationId.GetId() + " priority="
+ priority.GetPriority() + " resourceName=" + resourceName + " capability=" + capability
+ " numContainers=" + request.GetNumContainers() + " #asks=" + ask.Count);
}
}
}
public virtual void Move(Queue newQueue)
{
lock (this)
{
QueueMetrics oldMetrics = queue.GetMetrics();
QueueMetrics newMetrics = newQueue.GetMetrics();
foreach (IDictionary <string, ResourceRequest> asks in requests.Values)
{
ResourceRequest request = asks[ResourceRequest.Any];
if (request != null)
{
oldMetrics.DecrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
newMetrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
}
}
oldMetrics.MoveAppFrom(this);
newMetrics.MoveAppTo(this);
activeUsersManager.DeactivateApplication(user, applicationId);
activeUsersManager = newQueue.GetActiveUsersManager();
activeUsersManager.ActivateApplication(user, applicationId);
this.queue = newQueue;
this.queueName = newQueue.GetQueueName();
}
}
private void DecResourceRequest(string resourceName, Priority priority, ResourceRequest
request)
{
request.SetNumContainers(request.GetNumContainers() - 1);
if (request.GetNumContainers() == 0)
{
Sharpen.Collections.Remove(requests[priority], resourceName);
}
}
private void DecResourceRequest(Priority priority, string resourceName, Resource
capability)
{
IDictionary <string, IDictionary <Resource, ResourceRequest> > remoteRequests = this
.remoteRequestsTable[priority];
IDictionary <Resource, ResourceRequest> reqMap = remoteRequests[resourceName];
if (reqMap == null)
{
// as we modify the resource requests by filtering out blacklisted hosts
// when they are added, this value may be null when being
// decremented
if (Log.IsDebugEnabled())
{
Log.Debug("Not decrementing resource as " + resourceName + " is not present in request table"
);
}
return;
}
ResourceRequest remoteRequest = reqMap[capability];
if (Log.IsDebugEnabled())
{
Log.Debug("BEFORE decResourceRequest:" + " applicationId=" + applicationId.GetId(
) + " priority=" + priority.GetPriority() + " resourceName=" + resourceName + " numContainers="
+ remoteRequest.GetNumContainers() + " #asks=" + ask.Count);
}
if (remoteRequest.GetNumContainers() > 0)
{
// based on blacklisting comments above we can end up decrementing more
// than requested. so guard for that.
remoteRequest.SetNumContainers(remoteRequest.GetNumContainers() - 1);
}
if (remoteRequest.GetNumContainers() == 0)
{
Sharpen.Collections.Remove(reqMap, capability);
if (reqMap.Count == 0)
{
Sharpen.Collections.Remove(remoteRequests, resourceName);
}
if (remoteRequests.Count == 0)
{
Sharpen.Collections.Remove(remoteRequestsTable, priority);
}
}
// send the updated resource request to RM
// send 0 container count requests also to cancel previous requests
AddResourceRequestToAsk(remoteRequest);
if (Log.IsDebugEnabled())
{
Log.Info("AFTER decResourceRequest:" + " applicationId=" + applicationId.GetId()
+ " priority=" + priority.GetPriority() + " resourceName=" + resourceName + " numContainers="
+ remoteRequest.GetNumContainers() + " #asks=" + ask.Count);
}
}
private void ApplyRequestLimits()
{
IEnumerator <ResourceRequest> iter = requestLimits.Values.GetEnumerator();
while (iter.HasNext())
{
ResourceRequest reqLimit = iter.Next();
int limit = reqLimit.GetNumContainers();
IDictionary <string, IDictionary <Resource, ResourceRequest> > remoteRequests = remoteRequestsTable
[reqLimit.GetPriority()];
IDictionary <Resource, ResourceRequest> reqMap = (remoteRequests != null) ? remoteRequests
[ResourceRequest.Any] : null;
ResourceRequest req = (reqMap != null) ? reqMap[reqLimit.GetCapability()] : null;
if (req == null)
{
continue;
}
// update an existing ask or send a new one if updating
if (ask.Remove(req) || requestLimitsToUpdate.Contains(req))
{
ResourceRequest newReq = req.GetNumContainers() > limit ? reqLimit : req;
ask.AddItem(newReq);
Log.Info("Applying ask limit of " + newReq.GetNumContainers() + " for priority:"
+ reqLimit.GetPriority() + " and capability:" + reqLimit.GetCapability());
}
if (limit == int.MaxValue)
{
iter.Remove();
}
}
requestLimitsToUpdate.Clear();
}
private void AddResourceRequest(Priority priority, string resourceName, Resource
capability)
{
IDictionary <string, IDictionary <Resource, ResourceRequest> > remoteRequests = this
.remoteRequestsTable[priority];
if (remoteRequests == null)
{
remoteRequests = new Dictionary <string, IDictionary <Resource, ResourceRequest> >();
this.remoteRequestsTable[priority] = remoteRequests;
if (Log.IsDebugEnabled())
{
Log.Debug("Added priority=" + priority);
}
}
IDictionary <Resource, ResourceRequest> reqMap = remoteRequests[resourceName];
if (reqMap == null)
{
reqMap = new Dictionary <Resource, ResourceRequest>();
remoteRequests[resourceName] = reqMap;
}
ResourceRequest remoteRequest = reqMap[capability];
if (remoteRequest == null)
{
remoteRequest = recordFactory.NewRecordInstance <ResourceRequest>();
remoteRequest.SetPriority(priority);
remoteRequest.SetResourceName(resourceName);
remoteRequest.SetCapability(capability);
remoteRequest.SetNumContainers(0);
reqMap[capability] = remoteRequest;
}
remoteRequest.SetNumContainers(remoteRequest.GetNumContainers() + 1);
// Note this down for next interaction with ResourceManager
AddResourceRequestToAsk(remoteRequest);
if (Log.IsDebugEnabled())
{
Log.Debug("addResourceRequest:" + " applicationId=" + applicationId.GetId() + " priority="
+ priority.GetPriority() + " resourceName=" + resourceName + " numContainers="
+ remoteRequest.GetNumContainers() + " #asks=" + ask.Count);
}
}
public static ResourceRequest NewResourceRequest(ResourceRequest r)
{
ResourceRequest request = recordFactory.NewRecordInstance <ResourceRequest>();
request.SetPriority(r.GetPriority());
request.SetResourceName(r.GetResourceName());
request.SetCapability(r.GetCapability());
request.SetNumContainers(r.GetNumContainers());
return(request);
}
private void VerifyResourceRequest(AMRMClientImpl <AMRMClient.ContainerRequest> client
, AMRMClient.ContainerRequest request, string location, bool expectedRelaxLocality
)
{
ResourceRequest ask = client.remoteRequestsTable[request.GetPriority()][location]
[request.GetCapability()].remoteRequest;
NUnit.Framework.Assert.AreEqual(location, ask.GetResourceName());
NUnit.Framework.Assert.AreEqual(1, ask.GetNumContainers());
NUnit.Framework.Assert.AreEqual(expectedRelaxLocality, ask.GetRelaxLocality());
}
/// <summary>
/// Whether this app has containers requests that could be satisfied on the
/// given node, if the node had full space.
/// </summary>
public virtual bool HasContainerForNode(Priority prio, FSSchedulerNode node)
{
ResourceRequest anyRequest = GetResourceRequest(prio, ResourceRequest.Any);
ResourceRequest rackRequest = GetResourceRequest(prio, node.GetRackName());
ResourceRequest nodeRequest = GetResourceRequest(prio, node.GetNodeName());
return(anyRequest != null && anyRequest.GetNumContainers() > 0 && (anyRequest.GetRelaxLocality
() || (rackRequest != null && rackRequest.GetNumContainers() > 0)) && (rackRequest
== null || rackRequest.GetRelaxLocality() || (nodeRequest != null && nodeRequest
.GetNumContainers() > 0)) && Resources.LessThanOrEqual(ResourceCalculator, null,
anyRequest.GetCapability(), node.GetRMNode().GetTotalCapability()));
}
private void UpdateResourceRequest(ResourceRequest request)
{
request.SetNumContainers(request.GetNumContainers() - 1);
// Note this for next interaction with ResourceManager
ask.Remove(request);
ask.AddItem(BuilderUtils.NewResourceRequest(request));
// clone to ensure the RM doesn't manipulate the same obj
if (Log.IsDebugEnabled())
{
Log.Debug("updateResourceRequest:" + " application=" + applicationId + " request="
+ request);
}
}
private int AssignOffSwitchContainers(FiCaSchedulerNode node, FiCaSchedulerApp application
, Priority priority)
{
int assignedContainers = 0;
ResourceRequest request = application.GetResourceRequest(priority, ResourceRequest
.Any);
if (request != null)
{
assignedContainers = AssignContainer(node, application, priority, request.GetNumContainers
(), request, NodeType.OffSwitch);
}
return(assignedContainers);
}
private void DecrementOutstanding(ResourceRequest offSwitchRequest)
{
lock (this)
{
int numOffSwitchContainers = offSwitchRequest.GetNumContainers() - 1;
// Do not remove ANY
offSwitchRequest.SetNumContainers(numOffSwitchContainers);
// Do we have any outstanding requests?
// If there is nothing, we need to deactivate this application
if (numOffSwitchContainers == 0)
{
CheckForDeactivation();
}
}
}
protected internal virtual void SetRequestLimit(Priority priority, Resource capability
, int limit)
{
if (limit < 0)
{
limit = int.MaxValue;
}
ResourceRequest newReqLimit = ResourceRequest.NewInstance(priority, ResourceRequest
.Any, capability, limit);
ResourceRequest oldReqLimit = requestLimits[newReqLimit] = newReqLimit;
if (oldReqLimit == null || oldReqLimit.GetNumContainers() < limit)
{
requestLimitsToUpdate.AddItem(newReqLimit);
}
}
public virtual void Stop(RMAppAttemptState rmAppAttemptFinalState)
{
lock (this)
{
// clear pending resources metrics for the application
QueueMetrics metrics = queue.GetMetrics();
foreach (IDictionary <string, ResourceRequest> asks in requests.Values)
{
ResourceRequest request = asks[ResourceRequest.Any];
if (request != null)
{
metrics.DecrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
}
}
metrics.FinishAppAttempt(applicationId, pending, user);
// Clear requests themselves
ClearRequests();
}
}
private int AssignNodeLocalContainers(FiCaSchedulerNode node, FiCaSchedulerApp application
, Priority priority)
{
int assignedContainers = 0;
ResourceRequest request = application.GetResourceRequest(priority, node.GetNodeName
());
if (request != null)
{
// Don't allocate on this node if we don't need containers on this rack
ResourceRequest rackRequest = application.GetResourceRequest(priority, node.GetRMNode
().GetRackName());
if (rackRequest == null || rackRequest.GetNumContainers() <= 0)
{
return(0);
}
int assignableContainers = Math.Min(GetMaxAllocatableContainers(application, priority
, node, NodeType.NodeLocal), request.GetNumContainers());
assignedContainers = AssignContainer(node, application, priority, assignableContainers
, request, NodeType.NodeLocal);
}
return(assignedContainers);
}
private void CheckForDeactivation()
{
lock (this)
{
bool deactivate = true;
foreach (Priority priority in GetPriorities())
{
ResourceRequest request = GetResourceRequest(priority, ResourceRequest.Any);
if (request != null)
{
if (request.GetNumContainers() > 0)
{
deactivate = false;
break;
}
}
}
if (deactivate)
{
activeUsersManager.DeactivateApplication(user, applicationId);
}
}
}
private int AssignRackLocalContainers(FiCaSchedulerNode node, FiCaSchedulerApp application
, Priority priority)
{
int assignedContainers = 0;
ResourceRequest request = application.GetResourceRequest(priority, node.GetRMNode
().GetRackName());
if (request != null)
{
// Don't allocate on this rack if the application doens't need containers
ResourceRequest offSwitchRequest = application.GetResourceRequest(priority, ResourceRequest
.Any);
if (offSwitchRequest.GetNumContainers() <= 0)
{
return(0);
}
int assignableContainers = Math.Min(GetMaxAllocatableContainers(application, priority
, node, NodeType.RackLocal), request.GetNumContainers());
assignedContainers = AssignContainer(node, application, priority, assignableContainers
, request, NodeType.RackLocal);
}
return(assignedContainers);
}
private int GetMaxAllocatableContainers(FiCaSchedulerApp application, Priority priority
, FiCaSchedulerNode node, NodeType type)
{
int maxContainers = 0;
ResourceRequest offSwitchRequest = application.GetResourceRequest(priority, ResourceRequest
.Any);
if (offSwitchRequest != null)
{
maxContainers = offSwitchRequest.GetNumContainers();
}
if (type == NodeType.OffSwitch)
{
return(maxContainers);
}
if (type == NodeType.RackLocal)
{
ResourceRequest rackLocalRequest = application.GetResourceRequest(priority, node.
GetRMNode().GetRackName());
if (rackLocalRequest == null)
{
return(maxContainers);
}
maxContainers = Math.Min(maxContainers, rackLocalRequest.GetNumContainers());
}
if (type == NodeType.NodeLocal)
{
ResourceRequest nodeLocalRequest = application.GetResourceRequest(priority, node.
GetRMNode().GetNodeAddress());
if (nodeLocalRequest != null)
{
maxContainers = Math.Min(maxContainers, nodeLocalRequest.GetNumContainers());
}
}
return(maxContainers);
}
private Org.Apache.Hadoop.Yarn.Api.Records.Resource AssignContainer(FSSchedulerNode
node, bool reserved)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Node offered to app: " + GetName() + " reserved: " + reserved);
}
ICollection <Priority> prioritiesToTry = (reserved) ? Arrays.AsList(node.GetReservedContainer
().GetReservedPriority()) : GetPriorities();
// For each priority, see if we can schedule a node local, rack local
// or off-switch request. Rack of off-switch requests may be delayed
// (not scheduled) in order to promote better locality.
lock (this)
{
foreach (Priority priority in prioritiesToTry)
{
if (GetTotalRequiredResources(priority) <= 0 || !HasContainerForNode(priority, node
))
{
continue;
}
AddSchedulingOpportunity(priority);
// Check the AM resource usage for the leaf queue
if (GetLiveContainers().Count == 0 && !GetUnmanagedAM())
{
if (!((FSLeafQueue)GetQueue()).CanRunAppAM(GetAMResource()))
{
return(Resources.None());
}
}
ResourceRequest rackLocalRequest = GetResourceRequest(priority, node.GetRackName(
));
ResourceRequest localRequest = GetResourceRequest(priority, node.GetNodeName());
if (localRequest != null && !localRequest.GetRelaxLocality())
{
Log.Warn("Relax locality off is not supported on local request: " + localRequest);
}
NodeType allowedLocality;
if (scheduler.IsContinuousSchedulingEnabled())
{
allowedLocality = GetAllowedLocalityLevelByTime(priority, scheduler.GetNodeLocalityDelayMs
(), scheduler.GetRackLocalityDelayMs(), scheduler.GetClock().GetTime());
}
else
{
allowedLocality = GetAllowedLocalityLevel(priority, scheduler.GetNumClusterNodes(
), scheduler.GetNodeLocalityThreshold(), scheduler.GetRackLocalityThreshold());
}
if (rackLocalRequest != null && rackLocalRequest.GetNumContainers() != 0 && localRequest
!= null && localRequest.GetNumContainers() != 0)
{
return(AssignContainer(node, localRequest, NodeType.NodeLocal, reserved));
}
if (rackLocalRequest != null && !rackLocalRequest.GetRelaxLocality())
{
continue;
}
if (rackLocalRequest != null && rackLocalRequest.GetNumContainers() != 0 && (allowedLocality
.Equals(NodeType.RackLocal) || allowedLocality.Equals(NodeType.OffSwitch)))
{
return(AssignContainer(node, rackLocalRequest, NodeType.RackLocal, reserved));
}
ResourceRequest offSwitchRequest = GetResourceRequest(priority, ResourceRequest.Any
);
if (offSwitchRequest != null && !offSwitchRequest.GetRelaxLocality())
{
continue;
}
if (offSwitchRequest != null && offSwitchRequest.GetNumContainers() != 0)
{
if (!HasNodeOrRackLocalRequests(priority) || allowedLocality.Equals(NodeType.OffSwitch
))
{
return(AssignContainer(node, offSwitchRequest, NodeType.OffSwitch, reserved));
}
}
}
}
return(Resources.None());
}
/// <exception cref="System.Exception"/>
public virtual void TestUpdateResourceOnNode()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.RollMasterKey();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
FifoScheduler scheduler = new _FifoScheduler_275(this);
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration());
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(2048, 4), 1, "127.0.0.1"
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
scheduler.Handle(nodeEvent1);
MethodInfo method = Sharpen.Runtime.GetDeclaredMethod(scheduler.GetType(), "getNodes"
);
IDictionary <NodeId, FiCaSchedulerNode> schedulerNodes = (IDictionary <NodeId, FiCaSchedulerNode
>)method.Invoke(scheduler);
NUnit.Framework.Assert.AreEqual(schedulerNodes.Values.Count, 1);
Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = Resources.CreateResource
(1024, 4);
NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent
(node0, ResourceOption.NewInstance(newResource, RMNode.OverCommitTimeoutMillisDefault
));
scheduler.Handle(node0ResourceUpdate);
// SchedulerNode's total resource and available resource are changed.
NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetTotalResource
().GetMemory(), 1024);
NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetAvailableResource
().GetMemory(), 1024);
QueueInfo queueInfo = scheduler.GetQueueInfo(null, false, false);
NUnit.Framework.Assert.AreEqual(0.0f, queueInfo.GetCurrentCapacity(), 0.0f);
int _appId = 1;
int _appAttemptId = 1;
ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId);
CreateMockRMApp(appAttemptId, rmContext);
AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId
(), "queue1", "user1");
scheduler.Handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId
, false);
scheduler.Handle(attemptEvent);
int memory = 1024;
int priority = 1;
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority
, 1);
ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority
, 1);
ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority
, 1);
ask.AddItem(nodeLocal);
ask.AddItem(rackLocal);
ask.AddItem(any);
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
// Before the node update event, there are one local request
NUnit.Framework.Assert.AreEqual(1, nodeLocal.GetNumContainers());
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
// Now schedule.
scheduler.Handle(node0Update);
// After the node update event, check no local request
NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers());
// Also check that one container was scheduled
SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId);
NUnit.Framework.Assert.AreEqual(1, info.GetLiveContainers().Count);
// And check the default Queue now is full.
queueInfo = scheduler.GetQueueInfo(null, false, false);
NUnit.Framework.Assert.AreEqual(1.0f, queueInfo.GetCurrentCapacity(), 0.0f);
}
/// <summary>
/// The ApplicationMaster is updating resource requirements for the
/// application, by asking for more resources and releasing resources acquired
/// by the application.
/// </summary>
/// <param name="requests">resources to be acquired</param>
/// <param name="recoverPreemptedRequest">recover Resource Request on preemption</param>
public virtual void UpdateResourceRequests(IList <ResourceRequest> requests, bool
recoverPreemptedRequest)
{
lock (this)
{
QueueMetrics metrics = queue.GetMetrics();
// Update resource requests
foreach (ResourceRequest request in requests)
{
Priority priority = request.GetPriority();
string resourceName = request.GetResourceName();
bool updatePendingResources = false;
ResourceRequest lastRequest = null;
if (resourceName.Equals(ResourceRequest.Any))
{
if (Log.IsDebugEnabled())
{
Log.Debug("update:" + " application=" + applicationId + " request=" + request);
}
updatePendingResources = true;
// Premature optimization?
// Assumes that we won't see more than one priority request updated
// in one call, reasonable assumption... however, it's totally safe
// to activate same application more than once.
// Thus we don't need another loop ala the one in decrementOutstanding()
// which is needed during deactivate.
if (request.GetNumContainers() > 0)
{
activeUsersManager.ActivateApplication(user, applicationId);
}
}
IDictionary <string, ResourceRequest> asks = this.requests[priority];
if (asks == null)
{
asks = new ConcurrentHashMap <string, ResourceRequest>();
this.requests[priority] = asks;
this.priorities.AddItem(priority);
}
lastRequest = asks[resourceName];
if (recoverPreemptedRequest && lastRequest != null)
{
// Increment the number of containers to 1, as it is recovering a
// single container.
request.SetNumContainers(lastRequest.GetNumContainers() + 1);
}
asks[resourceName] = request;
if (updatePendingResources)
{
// Similarly, deactivate application?
if (request.GetNumContainers() <= 0)
{
Log.Info("checking for deactivate of application :" + this.applicationId);
CheckForDeactivation();
}
int lastRequestContainers = lastRequest != null?lastRequest.GetNumContainers() :
0;
Org.Apache.Hadoop.Yarn.Api.Records.Resource lastRequestCapability = lastRequest !=
null?lastRequest.GetCapability() : Resources.None();
metrics.IncrPendingResources(user, request.GetNumContainers(), request.GetCapability
());
metrics.DecrPendingResources(user, lastRequestContainers, lastRequestCapability);
}
}
}
}
/// <exception cref="System.Exception"/>
public virtual void TestNodeLocalAssignment()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.RollMasterKey();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
FifoScheduler scheduler = new FifoScheduler();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration());
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(1024 * 64), 1, "127.0.0.1"
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
scheduler.Handle(nodeEvent1);
int _appId = 1;
int _appAttemptId = 1;
ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId);
CreateMockRMApp(appAttemptId, rmContext);
AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId
(), "queue1", "user1");
scheduler.Handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId
, false);
scheduler.Handle(attemptEvent);
int memory = 64;
int nConts = 3;
int priority = 20;
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority
, nConts);
ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority
, nConts);
ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority
, nConts);
ask.AddItem(nodeLocal);
ask.AddItem(rackLocal);
ask.AddItem(any);
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
// Before the node update event, there are 3 local requests outstanding
NUnit.Framework.Assert.AreEqual(3, nodeLocal.GetNumContainers());
scheduler.Handle(node0Update);
// After the node update event, check that there are no more local requests
// outstanding
NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers());
//Also check that the containers were scheduled
SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId);
NUnit.Framework.Assert.AreEqual(3, info.GetLiveContainers().Count);
scheduler.Stop();
}
