protected internal virtual void SetupDispatcher(RMContext rmContext, Configuration
conf)
{
TestAppManager.TestDispatcher testDispatcher = new TestAppManager.TestDispatcher(
this);
TestAppManager.TestAppManagerDispatcher testAppManagerDispatcher = new TestAppManager.TestAppManagerDispatcher
(this);
rmContext.GetDispatcher().Register(typeof(RMAppEventType), testDispatcher);
rmContext.GetDispatcher().Register(typeof(RMAppManagerEventType), testAppManagerDispatcher
);
((Org.Apache.Hadoop.Service.Service)rmContext.GetDispatcher()).Init(conf);
((Org.Apache.Hadoop.Service.Service)rmContext.GetDispatcher()).Start();
NUnit.Framework.Assert.AreEqual("app event type is wrong before", RMAppEventType.
Kill, appEventType);
}
public AMLauncher(RMContext rmContext, RMAppAttempt application, AMLauncherEventType
eventType, Configuration conf)
{
this.application = application;
this.conf = conf;
this.eventType = eventType;
this.rmContext = rmContext;
this.handler = rmContext.GetDispatcher().GetEventHandler();
this.masterContainer = application.GetMasterContainer();
}
public virtual void ContainerLaunchedOnNode(ContainerId containerId, NodeId nodeId
)
{
lock (this)
{
// Inform the container
RMContainer rmContainer = GetRMContainer(containerId);
if (rmContainer == null)
{
// Some unknown container sneaked into the system. Kill it.
rmContext.GetDispatcher().GetEventHandler().Handle(new RMNodeCleanContainerEvent(
nodeId, containerId));
return;
}
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
));
}
}
public RMContainerImpl(Container container, ApplicationAttemptId appAttemptId, NodeId
nodeId, string user, RMContext rmContext, long creationTime)
{
// Transitions from NEW state
// Transitions from RESERVED state
// nothing to do
// nothing to do
// Transitions from ALLOCATED state
// Transitions from ACQUIRED state
// Transitions from RUNNING state
// Transitions from COMPLETED state
// Transitions from EXPIRED state
// Transitions from RELEASED state
// Transitions from KILLED state
// create the topology tables
this.stateMachine = stateMachineFactory.Make(this);
this.containerId = container.GetId();
this.nodeId = nodeId;
this.container = container;
this.appAttemptId = appAttemptId;
this.user = user;
this.creationTime = creationTime;
this.rmContext = rmContext;
this.eventHandler = rmContext.GetDispatcher().GetEventHandler();
this.containerAllocationExpirer = rmContext.GetContainerAllocationExpirer();
this.isAMContainer = false;
this.resourceRequests = null;
ReentrantReadWriteLock Lock = new ReentrantReadWriteLock();
this.readLock = Lock.ReadLock();
this.writeLock = Lock.WriteLock();
saveNonAMContainerMetaInfo = rmContext.GetYarnConfiguration().GetBoolean(YarnConfiguration
.ApplicationHistorySaveNonAmContainerMetaInfo, YarnConfiguration.DefaultApplicationHistorySaveNonAmContainerMetaInfo
);
rmContext.GetRMApplicationHistoryWriter().ContainerStarted(this);
// If saveNonAMContainerMetaInfo is true, store system metrics for all
// containers. If false, and if this container is marked as the AM, metrics
// will still be published for this container, but that calculation happens
// later.
if (saveNonAMContainerMetaInfo)
{
rmContext.GetSystemMetricsPublisher().ContainerCreated(this, this.creationTime);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TransitionToActive(HAServiceProtocol.StateChangeRequestInfo reqInfo
)
{
lock (this)
{
// call refreshAdminAcls before HA state transition
// for the case that adminAcls have been updated in previous active RM
try
{
RefreshAdminAcls(false);
}
catch (YarnException ex)
{
throw new ServiceFailedException("Can not execute refreshAdminAcls", ex);
}
UserGroupInformation user = CheckAccess("transitionToActive");
CheckHaStateChange(reqInfo);
try
{
rm.TransitionToActive();
}
catch (Exception e)
{
RMAuditLogger.LogFailure(user.GetShortUserName(), "transitionToActive", string.Empty
, "RMHAProtocolService", "Exception transitioning to active");
throw new ServiceFailedException("Error when transitioning to Active mode", e);
}
try
{
// call all refresh*s for active RM to get the updated configurations.
RefreshAll();
}
catch (Exception e)
{
Log.Error("RefreshAll failed so firing fatal event", e);
rmContext.GetDispatcher().GetEventHandler().Handle(new RMFatalEvent(RMFatalEventType
.TransitionToActiveFailed, e));
throw new ServiceFailedException("Error on refreshAll during transistion to Active"
, e);
}
RMAuditLogger.LogSuccess(user.GetShortUserName(), "transitionToActive", "RMHAProtocolService"
);
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
/// <exception cref="System.IO.IOException"/>
public virtual FinishApplicationMasterResponse FinishApplicationMaster(FinishApplicationMasterRequest
request)
{
ApplicationAttemptId applicationAttemptId = AuthorizeRequest().GetApplicationAttemptId
();
ApplicationId appId = applicationAttemptId.GetApplicationId();
RMApp rmApp = rmContext.GetRMApps()[applicationAttemptId.GetApplicationId()];
// checking whether the app exits in RMStateStore at first not to throw
// ApplicationDoesNotExistInCacheException before and after
// RM work-preserving restart.
if (rmApp.IsAppFinalStateStored())
{
Log.Info(rmApp.GetApplicationId() + " unregistered successfully. ");
return(FinishApplicationMasterResponse.NewInstance(true));
}
ApplicationMasterService.AllocateResponseLock Lock = responseMap[applicationAttemptId
];
if (Lock == null)
{
ThrowApplicationDoesNotExistInCacheException(applicationAttemptId);
}
// Allow only one thread in AM to do finishApp at a time.
lock (Lock)
{
if (!HasApplicationMasterRegistered(applicationAttemptId))
{
string message = "Application Master is trying to unregister before registering for: "
+ appId;
Log.Error(message);
RMAuditLogger.LogFailure(this.rmContext.GetRMApps()[appId].GetUser(), RMAuditLogger.AuditConstants
.UnregisterAm, string.Empty, "ApplicationMasterService", message, appId, applicationAttemptId
);
throw new ApplicationMasterNotRegisteredException(message);
}
this.amLivelinessMonitor.ReceivedPing(applicationAttemptId);
rmContext.GetDispatcher().GetEventHandler().Handle(new RMAppAttemptUnregistrationEvent
(applicationAttemptId, request.GetTrackingUrl(), request.GetFinalApplicationStatus
(), request.GetDiagnostics()));
// For UnmanagedAMs, return true so they don't retry
return(FinishApplicationMasterResponse.NewInstance(rmApp.GetApplicationSubmissionContext
().GetUnmanagedAM()));
}
}
internal virtual void HandleNMContainerStatus(NMContainerStatus containerStatus,
NodeId nodeId)
{
ApplicationAttemptId appAttemptId = containerStatus.GetContainerId().GetApplicationAttemptId
();
RMApp rmApp = rmContext.GetRMApps()[appAttemptId.GetApplicationId()];
if (rmApp == null)
{
Log.Error("Received finished container : " + containerStatus.GetContainerId() + " for unknown application "
+ appAttemptId.GetApplicationId() + " Skipping.");
return;
}
if (rmApp.GetApplicationSubmissionContext().GetUnmanagedAM())
{
if (Log.IsDebugEnabled())
{
Log.Debug("Ignoring container completion status for unmanaged AM " + rmApp.GetApplicationId
());
}
return;
}
RMAppAttempt rmAppAttempt = rmApp.GetRMAppAttempt(appAttemptId);
Container masterContainer = rmAppAttempt.GetMasterContainer();
if (masterContainer.GetId().Equals(containerStatus.GetContainerId()) && containerStatus
.GetContainerState() == ContainerState.Complete)
{
ContainerStatus status = ContainerStatus.NewInstance(containerStatus.GetContainerId
(), containerStatus.GetContainerState(), containerStatus.GetDiagnostics(), containerStatus
.GetContainerExitStatus());
// sending master container finished event.
RMAppAttemptContainerFinishedEvent evt = new RMAppAttemptContainerFinishedEvent(appAttemptId
, status, nodeId);
rmContext.GetDispatcher().GetEventHandler().Handle(evt);
}
}
public virtual void NotifyFatalError(string errorMessage)
{
rmContext.GetDispatcher().GetEventHandler().Handle(new RMFatalEvent(RMFatalEventType
.EmbeddedElectorFailed, errorMessage));
}
private void UpdateResourceMappings(IDictionary <string, CommonNodeLabelsManager.Host
> before, IDictionary <string, CommonNodeLabelsManager.Host> after)
{
// Get NMs in before only
ICollection <NodeId> allNMs = new HashSet <NodeId>();
foreach (KeyValuePair <string, CommonNodeLabelsManager.Host> entry in before)
{
Sharpen.Collections.AddAll(allNMs, entry.Value.nms.Keys);
}
foreach (KeyValuePair <string, CommonNodeLabelsManager.Host> entry_1 in after)
{
Sharpen.Collections.AddAll(allNMs, entry_1.Value.nms.Keys);
}
// Map used to notify RM
IDictionary <NodeId, ICollection <string> > newNodeToLabelsMap = new Dictionary <NodeId
, ICollection <string> >();
// traverse all nms
foreach (NodeId nodeId in allNMs)
{
CommonNodeLabelsManager.Node oldNM;
if ((oldNM = GetNMInNodeSet(nodeId, before, true)) != null)
{
ICollection <string> oldLabels = GetLabelsByNode(nodeId, before);
// no label in the past
if (oldLabels.IsEmpty())
{
// update labels
NodeLabel label = labelCollections[NoLabel];
label.RemoveNode(oldNM.resource);
// update queues, all queue can access this node
foreach (RMNodeLabelsManager.Queue q in queueCollections.Values)
{
Resources.SubtractFrom(q.resource, oldNM.resource);
}
}
else
{
// update labels
foreach (string labelName in oldLabels)
{
NodeLabel label = labelCollections[labelName];
if (null == label)
{
continue;
}
label.RemoveNode(oldNM.resource);
}
// update queues, only queue can access this node will be subtract
foreach (RMNodeLabelsManager.Queue q in queueCollections.Values)
{
if (IsNodeUsableByQueue(oldLabels, q))
{
Resources.SubtractFrom(q.resource, oldNM.resource);
}
}
}
}
CommonNodeLabelsManager.Node newNM;
if ((newNM = GetNMInNodeSet(nodeId, after, true)) != null)
{
ICollection <string> newLabels = GetLabelsByNode(nodeId, after);
newNodeToLabelsMap[nodeId] = ImmutableSet.CopyOf(newLabels);
// no label in the past
if (newLabels.IsEmpty())
{
// update labels
NodeLabel label = labelCollections[NoLabel];
label.AddNode(newNM.resource);
// update queues, all queue can access this node
foreach (RMNodeLabelsManager.Queue q in queueCollections.Values)
{
Resources.AddTo(q.resource, newNM.resource);
}
}
else
{
// update labels
foreach (string labelName in newLabels)
{
NodeLabel label = labelCollections[labelName];
label.AddNode(newNM.resource);
}
// update queues, only queue can access this node will be subtract
foreach (RMNodeLabelsManager.Queue q in queueCollections.Values)
{
if (IsNodeUsableByQueue(newLabels, q))
{
Resources.AddTo(q.resource, newNM.resource);
}
}
}
}
}
// Notify RM
if (rmContext != null && rmContext.GetDispatcher() != null)
{
rmContext.GetDispatcher().GetEventHandler().Handle(new NodeLabelsUpdateSchedulerEvent
(newNodeToLabelsMap));
}
}
public virtual void TestReleaseWhileRunning()
{
DrainDispatcher drainDispatcher = new DrainDispatcher();
EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
<EventHandler>();
EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();
drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
drainDispatcher.Register(typeof(RMNodeEventType), generic);
drainDispatcher.Init(new YarnConfiguration());
drainDispatcher.Start();
NodeId nodeId = BuilderUtils.NewNodeId("host", 3425);
ApplicationId appId = BuilderUtils.NewApplicationId(1, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
>();
Resource resource = BuilderUtils.NewResource(512, 1);
Priority priority = BuilderUtils.NewPriority(5);
Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
resource, priority, null);
ConcurrentMap <ApplicationId, RMApp> rmApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
())).ThenReturn(null);
Org.Mockito.Mockito.DoReturn(rmApp).When(rmApps)[(ApplicationId)Matchers.Any()];
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>();
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
);
Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
);
Org.Mockito.Mockito.When(rmContext.GetRMApps()).ThenReturn(rmApps);
Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
);
Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
, rmContext);
NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
), Matchers.AnyLong());
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
));
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
);
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
, rmContainer.GetLogURL());
// In RUNNING state. Verify RELEASED and associated actions.
Org.Mockito.Mockito.Reset(appAttemptEventHandler);
ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
, SchedulerUtils.ReleasedContainer);
rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
.Released));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
);
NUnit.Framework.Assert.AreEqual(SchedulerUtils.ReleasedContainer, rmContainer.GetDiagnosticsInfo
());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, rmContainer.GetContainerExitStatus
());
NUnit.Framework.Assert.AreEqual(ContainerState.Complete, rmContainer.GetContainerState
());
Org.Mockito.Mockito.Verify(writer).ContainerFinished(Matchers.Any <RMContainer>());
Org.Mockito.Mockito.Verify(publisher).ContainerFinished(Matchers.Any <RMContainer>
(), Matchers.AnyLong());
ArgumentCaptor <RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor.ForClass
<RMAppAttemptContainerFinishedEvent>();
Org.Mockito.Mockito.Verify(appAttemptEventHandler).Handle(captor.Capture());
RMAppAttemptContainerFinishedEvent cfEvent = captor.GetValue();
NUnit.Framework.Assert.AreEqual(appAttemptId, cfEvent.GetApplicationAttemptId());
NUnit.Framework.Assert.AreEqual(containerStatus, cfEvent.GetContainerStatus());
NUnit.Framework.Assert.AreEqual(RMAppAttemptEventType.ContainerFinished, cfEvent.
GetType());
// In RELEASED state. A FINIHSED event may come in.
rmContainer.Handle(new RMContainerFinishedEvent(containerId, SchedulerUtils.CreateAbnormalContainerStatus
(containerId, "FinishedContainer"), RMContainerEventType.Finished));
NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
);
}
public virtual void TestExpireWhileRunning()
{
DrainDispatcher drainDispatcher = new DrainDispatcher();
EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
<EventHandler>();
EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();
drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
drainDispatcher.Register(typeof(RMNodeEventType), generic);
drainDispatcher.Init(new YarnConfiguration());
drainDispatcher.Start();
NodeId nodeId = BuilderUtils.NewNodeId("host", 3425);
ApplicationId appId = BuilderUtils.NewApplicationId(1, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
>();
Resource resource = BuilderUtils.NewResource(512, 1);
Priority priority = BuilderUtils.NewPriority(5);
Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
resource, priority, null);
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>();
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
);
Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
);
Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
);
Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
, rmContext);
NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
), Matchers.AnyLong());
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
));
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
);
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
, rmContainer.GetLogURL());
// In RUNNING state. Verify EXPIRE and associated actions.
Org.Mockito.Mockito.Reset(appAttemptEventHandler);
ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
, SchedulerUtils.ExpiredContainer);
rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
.Expire));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
Org.Mockito.Mockito.Verify(writer, Org.Mockito.Mockito.Never()).ContainerFinished
(Matchers.Any <RMContainer>());
Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Never()).ContainerFinished
(Matchers.Any <RMContainer>(), Matchers.AnyLong());
}
public virtual void TestSortedQueues()
{
// Setup queue configs
SetupSortedQueues(csConf);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
.Root, queues, queues, TestUtils.spyHook);
// Setup some nodes
int memoryPerNode = 10;
int coresPerNode = 16;
int numNodes = 1;
FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode
* Gb);
Org.Mockito.Mockito.DoNothing().When(node_0).ReleaseContainer(Matchers.Any <Container
>());
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(numNodes * (memoryPerNode * Gb), numNodes * coresPerNode);
Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes);
// Start testing
CSQueue a = queues[A];
CSQueue b = queues[B];
CSQueue c = queues[C];
CSQueue d = queues[D];
string user_0 = "user_0";
// Stub an App and its containerCompleted
FiCaSchedulerApp app_0 = GetMockApplication(0, user_0);
Org.Mockito.Mockito.DoReturn(true).When(app_0).ContainerCompleted(Matchers.Any <RMContainer
>(), Matchers.Any <ContainerStatus>(), Matchers.Any <RMContainerEventType>());
Priority priority = TestUtils.CreateMockPriority(1);
ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
>();
DrainDispatcher drainDispatcher = new DrainDispatcher();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>();
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
);
Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
);
Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
);
Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
());
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app_0.GetApplicationId
(), 1);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
Container container = TestUtils.GetMockContainer(containerId, node_0.GetNodeID(),
Resources.CreateResource(1 * Gb), priority);
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, node_0.GetNodeID
(), "user", rmContext);
// Assign {1,2,3,4} 1GB containers respectively to queues
StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
for (int i = 0; i < 2; i++)
{
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
for (int i_1 = 0; i_1 < 3; i_1++)
{
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
for (int i_2 = 0; i_2 < 4; i_2++)
{
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 1 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
VerifyQueueMetrics(a, 1 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 4 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
//Release 3 x 1GB containers from D
for (int i_3 = 0; i_3 < 3; i_3++)
{
d.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
.Kill, null, true);
}
VerifyQueueMetrics(a, 1 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
//reset manually resources on node
node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 1 - 2 -
3 - 1) * Gb);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Assign 2 x 1GB Containers to A
for (int i_4 = 0; i_4 < 2; i_4++)
{
StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
//Release 1GB Container from A
a.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
.Kill, null, true);
VerifyQueueMetrics(a, 2 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
//reset manually resources on node
node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 -
3 - 1) * Gb);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Assign 1GB container to B
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(a, 2 * Gb, clusterResource);
VerifyQueueMetrics(b, 3 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
//Release 1GB container resources from B
b.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
.Kill, null, true);
VerifyQueueMetrics(a, 2 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
//reset manually resources on node
node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 -
3 - 1) * Gb);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Assign 1GB container to A
StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Now do the real test, where B and D request a 1GB container
// D should should get the next container if the order is correct
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 1 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
InOrder allocationOrder = Org.Mockito.Mockito.InOrder(d, b);
allocationOrder.Verify(d).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>());
allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>());
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 2 * Gb, clusterResource);
//D got the container
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
}
public virtual void TearDown()
{
SetAppEventType(RMAppEventType.Kill);
((Org.Apache.Hadoop.Service.Service)rmContext.GetDispatcher()).Stop();
}