/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual StartContainersResponse StartContainers(StartContainersRequest requests
)
{
lock (this)
{
foreach (StartContainerRequest request in requests.GetStartContainerRequests())
{
Token containerToken = request.GetContainerToken();
ContainerTokenIdentifier tokenId = null;
try
{
tokenId = BuilderUtils.NewContainerTokenIdentifier(containerToken);
}
catch (IOException e)
{
throw RPCUtil.GetRemoteException(e);
}
ContainerId containerID = tokenId.GetContainerID();
ApplicationId applicationId = containerID.GetApplicationAttemptId().GetApplicationId
();
IList <Container> applicationContainers = containers[applicationId];
if (applicationContainers == null)
{
applicationContainers = new AList <Container>();
containers[applicationId] = applicationContainers;
}
// Sanity check
foreach (Container container in applicationContainers)
{
if (container.GetId().CompareTo(containerID) == 0)
{
throw new InvalidOperationException("Container " + containerID + " already setup on node "
+ containerManagerAddress);
}
}
Container container_1 = BuilderUtils.NewContainer(containerID, this.nodeId, nodeHttpAddress
, tokenId.GetResource(), null, null);
// DKDC - Doesn't matter
ContainerStatus containerStatus = BuilderUtils.NewContainerStatus(container_1.GetId
(), ContainerState.New, string.Empty, -1000);
applicationContainers.AddItem(container_1);
containerStatusMap[container_1] = containerStatus;
Resources.SubtractFrom(available, tokenId.GetResource());
Resources.AddTo(used, tokenId.GetResource());
if (Log.IsDebugEnabled())
{
Log.Debug("startContainer:" + " node=" + containerManagerAddress + " application="
+ applicationId + " container=" + container_1 + " available=" + available + " used="
+ used);
}
}
StartContainersResponse response = StartContainersResponse.NewInstance(null, null
, null);
return(response);
}
}
/// <summary>
/// Create and return a container object reflecting an allocation for the
/// given appliction on the given node with the given capability and
/// priority.
/// </summary>
public virtual Container CreateContainer(FSSchedulerNode node, Org.Apache.Hadoop.Yarn.Api.Records.Resource
capability, Priority priority)
{
NodeId nodeId = node.GetRMNode().GetNodeID();
ContainerId containerId = BuilderUtils.NewContainerId(GetApplicationAttemptId(),
GetNewContainerId());
// Create the container
Container container = BuilderUtils.NewContainer(containerId, nodeId, node.GetRMNode
().GetHttpAddress(), capability, priority, null);
return(container);
}
private int AssignContainer(FiCaSchedulerNode node, FiCaSchedulerApp application,
Priority priority, int assignableContainers, ResourceRequest request, NodeType type
)
{
Log.Debug("assignContainers:" + " node=" + node.GetRMNode().GetNodeAddress() + " application="
+ application.GetApplicationId().GetId() + " priority=" + priority.GetPriority(
) + " assignableContainers=" + assignableContainers + " request=" + request + " type="
+ type);
Org.Apache.Hadoop.Yarn.Api.Records.Resource capability = request.GetCapability();
int availableContainers = node.GetAvailableResource().GetMemory() / capability.GetMemory
();
// TODO: A buggy
// application
// with this
// zero would
// crash the
// scheduler.
int assignedContainers = Math.Min(assignableContainers, availableContainers);
if (assignedContainers > 0)
{
for (int i = 0; i < assignedContainers; ++i)
{
NodeId nodeId = node.GetRMNode().GetNodeID();
ContainerId containerId = BuilderUtils.NewContainerId(application.GetApplicationAttemptId
(), application.GetNewContainerId());
// Create the container
Container container = BuilderUtils.NewContainer(containerId, nodeId, node.GetRMNode
().GetHttpAddress(), capability, priority, null);
// Allocate!
// Inform the application
RMContainer rmContainer = application.Allocate(type, node, priority, request, container
);
// Inform the node
node.AllocateContainer(rmContainer);
// Update usage for this container
IncreaseUsedResources(rmContainer);
}
}
return(assignedContainers);
}
public virtual void TestReleaseWhileRunning()
{
DrainDispatcher drainDispatcher = new DrainDispatcher();
EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
<EventHandler>();
EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();
drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
drainDispatcher.Register(typeof(RMNodeEventType), generic);
drainDispatcher.Init(new YarnConfiguration());
drainDispatcher.Start();
NodeId nodeId = BuilderUtils.NewNodeId("host", 3425);
ApplicationId appId = BuilderUtils.NewApplicationId(1, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
>();
Resource resource = BuilderUtils.NewResource(512, 1);
Priority priority = BuilderUtils.NewPriority(5);
Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
resource, priority, null);
ConcurrentMap <ApplicationId, RMApp> rmApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
())).ThenReturn(null);
Org.Mockito.Mockito.DoReturn(rmApp).When(rmApps)[(ApplicationId)Matchers.Any()];
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>();
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
);
Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
);
Org.Mockito.Mockito.When(rmContext.GetRMApps()).ThenReturn(rmApps);
Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
);
Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
, rmContext);
NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
), Matchers.AnyLong());
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
));
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
);
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
, rmContainer.GetLogURL());
// In RUNNING state. Verify RELEASED and associated actions.
Org.Mockito.Mockito.Reset(appAttemptEventHandler);
ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
, SchedulerUtils.ReleasedContainer);
rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
.Released));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
);
NUnit.Framework.Assert.AreEqual(SchedulerUtils.ReleasedContainer, rmContainer.GetDiagnosticsInfo
());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, rmContainer.GetContainerExitStatus
());
NUnit.Framework.Assert.AreEqual(ContainerState.Complete, rmContainer.GetContainerState
());
Org.Mockito.Mockito.Verify(writer).ContainerFinished(Matchers.Any <RMContainer>());
Org.Mockito.Mockito.Verify(publisher).ContainerFinished(Matchers.Any <RMContainer>
(), Matchers.AnyLong());
ArgumentCaptor <RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor.ForClass
<RMAppAttemptContainerFinishedEvent>();
Org.Mockito.Mockito.Verify(appAttemptEventHandler).Handle(captor.Capture());
RMAppAttemptContainerFinishedEvent cfEvent = captor.GetValue();
NUnit.Framework.Assert.AreEqual(appAttemptId, cfEvent.GetApplicationAttemptId());
NUnit.Framework.Assert.AreEqual(containerStatus, cfEvent.GetContainerStatus());
NUnit.Framework.Assert.AreEqual(RMAppAttemptEventType.ContainerFinished, cfEvent.
GetType());
// In RELEASED state. A FINIHSED event may come in.
rmContainer.Handle(new RMContainerFinishedEvent(containerId, SchedulerUtils.CreateAbnormalContainerStatus
(containerId, "FinishedContainer"), RMContainerEventType.Finished));
NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
);
}
public virtual void TestExpireWhileRunning()
{
DrainDispatcher drainDispatcher = new DrainDispatcher();
EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
<EventHandler>();
EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();
drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
drainDispatcher.Register(typeof(RMNodeEventType), generic);
drainDispatcher.Init(new YarnConfiguration());
drainDispatcher.Start();
NodeId nodeId = BuilderUtils.NewNodeId("host", 3425);
ApplicationId appId = BuilderUtils.NewApplicationId(1, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
>();
Resource resource = BuilderUtils.NewResource(512, 1);
Priority priority = BuilderUtils.NewPriority(5);
Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
resource, priority, null);
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>();
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
);
Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
);
Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
);
Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
, rmContext);
NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
), Matchers.AnyLong());
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
));
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
);
rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
, rmContainer.GetLogURL());
// In RUNNING state. Verify EXPIRE and associated actions.
Org.Mockito.Mockito.Reset(appAttemptEventHandler);
ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
, SchedulerUtils.ExpiredContainer);
rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
.Expire));
drainDispatcher.Await();
NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
Org.Mockito.Mockito.Verify(writer, Org.Mockito.Mockito.Never()).ContainerFinished
(Matchers.Any <RMContainer>());
Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Never()).ContainerFinished
(Matchers.Any <RMContainer>(), Matchers.AnyLong());
}