public NodeInfo(RMNode ni, ResourceScheduler sched)
{
// JAXB needs this
NodeId id = ni.GetNodeID();
SchedulerNodeReport report = sched.GetNodeReport(id);
this.numContainers = 0;
this.usedMemoryMB = 0;
this.availMemoryMB = 0;
if (report != null)
{
this.numContainers = report.GetNumContainers();
this.usedMemoryMB = report.GetUsedResource().GetMemory();
this.availMemoryMB = report.GetAvailableResource().GetMemory();
this.usedVirtualCores = report.GetUsedResource().GetVirtualCores();
this.availableVirtualCores = report.GetAvailableResource().GetVirtualCores();
}
this.id = id.ToString();
this.rack = ni.GetRackName();
this.nodeHostName = ni.GetHostName();
this.state = ni.GetState();
this.nodeHTTPAddress = ni.GetHttpAddress();
this.lastHealthUpdate = ni.GetLastHealthReportTime();
this.healthReport = ni.GetHealthReport().ToString();
this.version = ni.GetNodeManagerVersion();
// add labels
ICollection <string> labelSet = ni.GetNodeLabels();
if (labelSet != null)
{
Sharpen.Collections.AddAll(nodeLabels, labelSet);
nodeLabels.Sort();
}
}
public SchedulerNode(RMNode node, bool usePortForNodeName, ICollection <string> labels
)
{
/* set of containers that are allocated containers */
this.rmNode = node;
this.availableResource = Resources.Clone(node.GetTotalCapability());
this.totalResourceCapability = Resources.Clone(node.GetTotalCapability());
if (usePortForNodeName)
{
nodeName = rmNode.GetHostName() + ":" + node.GetNodeID().GetPort();
}
else
{
nodeName = rmNode.GetHostName();
}
this.labels = ImmutableSet.CopyOf(labels);
}
public static FiCaSchedulerNode GetMockNode(string host, string rack, int port, int
capability)
{
NodeId nodeId = Org.Mockito.Mockito.Mock <NodeId>();
Org.Mockito.Mockito.When(nodeId.GetHost()).ThenReturn(host);
Org.Mockito.Mockito.When(nodeId.GetPort()).ThenReturn(port);
RMNode rmNode = Org.Mockito.Mockito.Mock <RMNode>();
Org.Mockito.Mockito.When(rmNode.GetNodeID()).ThenReturn(nodeId);
Org.Mockito.Mockito.When(rmNode.GetTotalCapability()).ThenReturn(Resources.CreateResource
(capability, 1));
Org.Mockito.Mockito.When(rmNode.GetNodeAddress()).ThenReturn(host + ":" + port);
Org.Mockito.Mockito.When(rmNode.GetHostName()).ThenReturn(host);
Org.Mockito.Mockito.When(rmNode.GetRackName()).ThenReturn(rack);
FiCaSchedulerNode node = Org.Mockito.Mockito.Spy(new FiCaSchedulerNode(rmNode, false
));
Log.Info("node = " + host + " avail=" + node.GetAvailableResource());
return(node);
}
/// <exception cref="System.Exception"/>
public virtual void TestUpdateResourceOnNode()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.RollMasterKey();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
FifoScheduler scheduler = new _FifoScheduler_275(this);
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration());
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(2048, 4), 1, "127.0.0.1"
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
scheduler.Handle(nodeEvent1);
MethodInfo method = Sharpen.Runtime.GetDeclaredMethod(scheduler.GetType(), "getNodes"
);
IDictionary <NodeId, FiCaSchedulerNode> schedulerNodes = (IDictionary <NodeId, FiCaSchedulerNode
>)method.Invoke(scheduler);
NUnit.Framework.Assert.AreEqual(schedulerNodes.Values.Count, 1);
Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = Resources.CreateResource
(1024, 4);
NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent
(node0, ResourceOption.NewInstance(newResource, RMNode.OverCommitTimeoutMillisDefault
));
scheduler.Handle(node0ResourceUpdate);
// SchedulerNode's total resource and available resource are changed.
NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetTotalResource
().GetMemory(), 1024);
NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetAvailableResource
().GetMemory(), 1024);
QueueInfo queueInfo = scheduler.GetQueueInfo(null, false, false);
NUnit.Framework.Assert.AreEqual(0.0f, queueInfo.GetCurrentCapacity(), 0.0f);
int _appId = 1;
int _appAttemptId = 1;
ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId);
CreateMockRMApp(appAttemptId, rmContext);
AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId
(), "queue1", "user1");
scheduler.Handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId
, false);
scheduler.Handle(attemptEvent);
int memory = 1024;
int priority = 1;
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority
, 1);
ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority
, 1);
ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority
, 1);
ask.AddItem(nodeLocal);
ask.AddItem(rackLocal);
ask.AddItem(any);
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
// Before the node update event, there are one local request
NUnit.Framework.Assert.AreEqual(1, nodeLocal.GetNumContainers());
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
// Now schedule.
scheduler.Handle(node0Update);
// After the node update event, check no local request
NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers());
// Also check that one container was scheduled
SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId);
NUnit.Framework.Assert.AreEqual(1, info.GetLiveContainers().Count);
// And check the default Queue now is full.
queueInfo = scheduler.GetQueueInfo(null, false, false);
NUnit.Framework.Assert.AreEqual(1.0f, queueInfo.GetCurrentCapacity(), 0.0f);
}
/// <exception cref="System.Exception"/>
public virtual void TestNodeLocalAssignment()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.RollMasterKey();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
FifoScheduler scheduler = new FifoScheduler();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration());
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(1024 * 64), 1, "127.0.0.1"
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
scheduler.Handle(nodeEvent1);
int _appId = 1;
int _appAttemptId = 1;
ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId);
CreateMockRMApp(appAttemptId, rmContext);
AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId
(), "queue1", "user1");
scheduler.Handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId
, false);
scheduler.Handle(attemptEvent);
int memory = 64;
int nConts = 3;
int priority = 20;
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority
, nConts);
ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority
, nConts);
ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority
, nConts);
ask.AddItem(nodeLocal);
ask.AddItem(rackLocal);
ask.AddItem(any);
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
// Before the node update event, there are 3 local requests outstanding
NUnit.Framework.Assert.AreEqual(3, nodeLocal.GetNumContainers());
scheduler.Handle(node0Update);
// After the node update event, check that there are no more local requests
// outstanding
NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers());
//Also check that the containers were scheduled
SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId);
NUnit.Framework.Assert.AreEqual(3, info.GetLiveContainers().Count);
scheduler.Stop();
}
