/// <exception cref="System.Exception"/>
public virtual void TestSchedulingDelay()
{
// Add one node
string host = "127.0.0.1";
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(4096, 4), 1, host
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
NodeUpdateSchedulerEvent nodeUpdateEvent = new NodeUpdateSchedulerEvent(node1);
scheduler.Handle(nodeUpdateEvent);
// Create one application and submit one each of node-local, rack-local
// and ANY requests
ApplicationAttemptId appAttemptId = CreateAppAttemptId(this.AppId++, this.AttemptId
++);
CreateMockRMApp(appAttemptId);
scheduler.AddApplication(appAttemptId.GetApplicationId(), "queue11", "user11", false
);
scheduler.AddApplicationAttempt(appAttemptId, false, false);
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ask.AddItem(CreateResourceRequest(1024, 1, ResourceRequest.Any, 1, 1, true));
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
FSAppAttempt app = scheduler.GetSchedulerApp(appAttemptId);
// Advance time and let continuous scheduling kick in
mockClock.Tick(1);
while (1024 != app.GetCurrentConsumption().GetMemory())
{
Sharpen.Thread.Sleep(100);
}
NUnit.Framework.Assert.AreEqual(1024, app.GetCurrentConsumption().GetMemory());
}
public virtual void TestRMNMInfoMissmatch()
{
RMContext rmc = Org.Mockito.Mockito.Mock <RMContext>();
ResourceScheduler rms = Org.Mockito.Mockito.Mock <ResourceScheduler>();
ConcurrentMap <NodeId, RMNode> map = new ConcurrentHashMap <NodeId, RMNode>();
RMNode node = MockNodes.NewNodeInfo(1, MockNodes.NewResource(4 * 1024));
map[node.GetNodeID()] = node;
Org.Mockito.Mockito.When(rmc.GetRMNodes()).ThenReturn(map);
RMNMInfo rmInfo = new RMNMInfo(rmc, rms);
string liveNMs = rmInfo.GetLiveNodeManagers();
ObjectMapper mapper = new ObjectMapper();
JsonNode jn = mapper.ReadTree(liveNMs);
NUnit.Framework.Assert.AreEqual("Unexpected number of live nodes:", 1, jn.Size());
IEnumerator <JsonNode> it = jn.GetEnumerator();
while (it.HasNext())
{
JsonNode n = it.Next();
NUnit.Framework.Assert.IsNotNull(n.Get("HostName"));
NUnit.Framework.Assert.IsNotNull(n.Get("Rack"));
NUnit.Framework.Assert.IsTrue("Node " + n.Get("NodeId") + " should be RUNNING", n
.Get("State").AsText().Contains("RUNNING"));
NUnit.Framework.Assert.IsNotNull(n.Get("NodeHTTPAddress"));
NUnit.Framework.Assert.IsNotNull(n.Get("LastHealthUpdate"));
NUnit.Framework.Assert.IsNotNull(n.Get("HealthReport"));
NUnit.Framework.Assert.IsNotNull(n.Get("NodeManagerVersion"));
NUnit.Framework.Assert.IsNull(n.Get("NumContainers"));
NUnit.Framework.Assert.IsNull(n.Get("UsedMemoryMB"));
NUnit.Framework.Assert.IsNull(n.Get("AvailableMemoryMB"));
}
}
/// <exception cref="System.IO.IOException"/>
private void CreateClusterWithQueuesAndOneNode(int mem, int vCores, string policy
)
{
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine("<queue name=\"root\" >");
@out.WriteLine(" <queue name=\"parentA\" >");
@out.WriteLine(" <weight>8</weight>");
@out.WriteLine(" <queue name=\"childA1\" />");
@out.WriteLine(" <queue name=\"childA2\" />");
@out.WriteLine(" <queue name=\"childA3\" />");
@out.WriteLine(" <queue name=\"childA4\" />");
@out.WriteLine(" </queue>");
@out.WriteLine(" <queue name=\"parentB\" >");
@out.WriteLine(" <weight>1</weight>");
@out.WriteLine(" <queue name=\"childB1\" />");
@out.WriteLine(" <queue name=\"childB2\" />");
@out.WriteLine(" </queue>");
@out.WriteLine("</queue>");
@out.WriteLine("<defaultQueueSchedulingPolicy>" + policy + "</defaultQueueSchedulingPolicy>"
);
@out.WriteLine("</allocations>");
@out.Close();
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(mem, vCores), 1,
"127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
}
public static RMContext MockRMContext(int numApps, int racks, int numNodes, int mbsPerNode
)
{
IList <RMApp> apps = MockAsm.NewApplications(numApps);
ConcurrentMap <ApplicationId, RMApp> applicationsMaps = Maps.NewConcurrentMap();
foreach (RMApp app in apps)
{
applicationsMaps[app.GetApplicationId()] = app;
}
IList <RMNode> nodes = MockNodes.NewNodes(racks, numNodes, MockNodes.NewResource(mbsPerNode
));
ConcurrentMap <NodeId, RMNode> nodesMap = Maps.NewConcurrentMap();
foreach (RMNode node in nodes)
{
nodesMap[node.GetNodeID()] = node;
}
IList <RMNode> deactivatedNodes = MockNodes.DeactivatedNodes(racks, numNodes, MockNodes.NewResource
(mbsPerNode));
ConcurrentMap <string, RMNode> deactivatedNodesMap = Maps.NewConcurrentMap();
foreach (RMNode node_1 in deactivatedNodes)
{
deactivatedNodesMap[node_1.GetHostName()] = node_1;
}
RMContextImpl rmContext = new _RMContextImpl_183(applicationsMaps, deactivatedNodesMap
, nodesMap, null, null, null, null, null, null, null, null, null, null);
rmContext.SetNodeLabelManager(new NullRMNodeLabelsManager());
return(rmContext);
}
public virtual void TestMaxAllocationAfterUpdateNodeResource()
{
int configuredMaxVCores = 20;
int configuredMaxMemory = 10 * 1024;
Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(configuredMaxMemory, configuredMaxVCores);
ConfigureScheduler();
YarnConfiguration conf = GetConf();
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores
);
conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory
);
conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0);
MockRM rm = new MockRM(conf);
try
{
rm.Start();
AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler(
);
VerifyMaximumResourceCapability(configuredMaximumResource, scheduler);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(2048, 5);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(4096, 10);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource3 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(512, 1);
Org.Apache.Hadoop.Yarn.Api.Records.Resource resource4 = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(1024, 2);
RMNode node1 = MockNodes.NewNodeInfo(0, resource1, 1, "127.0.0.2");
scheduler.Handle(new NodeAddedSchedulerEvent(node1));
RMNode node2 = MockNodes.NewNodeInfo(0, resource3, 2, "127.0.0.3");
scheduler.Handle(new NodeAddedSchedulerEvent(node2));
VerifyMaximumResourceCapability(resource1, scheduler);
// increase node1 resource
scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource2, 0));
VerifyMaximumResourceCapability(resource2, scheduler);
// decrease node1 resource
scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource1, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
// increase node2 resource
scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource4, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
// decrease node2 resource
scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource3, 0));
VerifyMaximumResourceCapability(resource1, scheduler);
}
finally
{
rm.Stop();
}
}
/// <exception cref="System.Exception"/>
public virtual void Test()
{
conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine("<queue name=\"queueA\">");
@out.WriteLine("<minResources>2048mb,0vcores</minResources>");
@out.WriteLine("</queue>");
@out.WriteLine("<queue name=\"queueB\">");
@out.WriteLine("<minResources>2048mb,0vcores</minResources>");
@out.WriteLine("</queue>");
@out.WriteLine("</allocations>");
@out.Close();
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
// Add one big node (only care about aggregate capacity)
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(4 * 1024, 4), 1,
"127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
scheduler.Update();
// Queue A wants 3 * 1024. Node update gives this all to A
CreateSchedulingRequest(3 * 1024, "queueA", "user1");
scheduler.Update();
NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);
scheduler.Handle(nodeEvent2);
// Queue B arrives and wants 1 * 1024
CreateSchedulingRequest(1 * 1024, "queueB", "user1");
scheduler.Update();
ICollection <FSLeafQueue> queues = scheduler.GetQueueManager().GetLeafQueues();
NUnit.Framework.Assert.AreEqual(3, queues.Count);
// Queue A should be above min share, B below.
FSLeafQueue queueA = scheduler.GetQueueManager().GetLeafQueue("queueA", false);
FSLeafQueue queueB = scheduler.GetQueueManager().GetLeafQueue("queueB", false);
NUnit.Framework.Assert.IsFalse(queueA.IsStarvedForMinShare());
NUnit.Framework.Assert.IsTrue(queueB.IsStarvedForMinShare());
// Node checks in again, should allocate for B
scheduler.Handle(nodeEvent2);
// Now B should have min share ( = demand here)
NUnit.Framework.Assert.IsFalse(queueB.IsStarvedForMinShare());
}
/// <exception cref="System.IO.IOException"/>
public static FairScheduler SetupFairScheduler(ReservationSystemTestUtil testUtil
, RMContext rmContext, Configuration conf, int numContainers)
{
FairScheduler scheduler = new FairScheduler();
scheduler.SetRMContext(rmContext);
Org.Mockito.Mockito.When(rmContext.GetScheduler()).ThenReturn(scheduler);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(conf, rmContext);
Resource resource = ReservationSystemTestUtil.CalculateClusterResource(numContainers
);
RMNode node1 = MockNodes.NewNodeInfo(1, resource, 1, "127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
return(scheduler);
}
/// <exception cref="System.Exception"/>
private void TestMaximumAllocationMemoryHelper(AbstractYarnScheduler scheduler, int
node1MaxMemory, int node2MaxMemory, int node3MaxMemory, params int[] expectedMaxMemory
)
{
NUnit.Framework.Assert.AreEqual(6, expectedMaxMemory.Length);
NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes());
int maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[0], maxMemory);
RMNode node1 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node1MaxMemory),
1, "127.0.0.2");
scheduler.Handle(new NodeAddedSchedulerEvent(node1));
NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[1], maxMemory);
scheduler.Handle(new NodeRemovedSchedulerEvent(node1));
NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[2], maxMemory);
RMNode node2 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node2MaxMemory),
2, "127.0.0.3");
scheduler.Handle(new NodeAddedSchedulerEvent(node2));
NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[3], maxMemory);
RMNode node3 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node3MaxMemory),
3, "127.0.0.4");
scheduler.Handle(new NodeAddedSchedulerEvent(node3));
NUnit.Framework.Assert.AreEqual(2, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[4], maxMemory);
scheduler.Handle(new NodeRemovedSchedulerEvent(node3));
NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes());
maxMemory = scheduler.GetMaximumResourceCapability().GetMemory();
NUnit.Framework.Assert.AreEqual(expectedMaxMemory[5], maxMemory);
scheduler.Handle(new NodeRemovedSchedulerEvent(node2));
NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes());
}
private void RegisterNodeAndSubmitApp(int memory, int vcores, int appContainers,
int appMemory)
{
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(memory, vcores),
1, "node1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
NUnit.Framework.Assert.AreEqual("Incorrect amount of resources in the cluster", memory
, scheduler.rootMetrics.GetAvailableMB());
NUnit.Framework.Assert.AreEqual("Incorrect amount of resources in the cluster", vcores
, scheduler.rootMetrics.GetAvailableVirtualCores());
CreateSchedulingRequest(appMemory, "queueA", "user1", appContainers);
scheduler.Update();
// Sufficient node check-ins to fully schedule containers
for (int i = 0; i < 3; i++)
{
NodeUpdateSchedulerEvent nodeUpdate1 = new NodeUpdateSchedulerEvent(node1);
scheduler.Handle(nodeUpdate1);
}
NUnit.Framework.Assert.AreEqual("app1's request is not met", memory - appContainers
* appMemory, scheduler.rootMetrics.GetAvailableMB());
}
public virtual void TestBlackListNodes()
{
Configuration conf = new Configuration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
MockRM rm = new MockRM(conf);
rm.Start();
FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
string host = "127.0.0.1";
RMNode node = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, host);
fs.Handle(new NodeAddedSchedulerEvent(node));
ApplicationId appId = BuilderUtils.NewApplicationId(100, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
CreateMockRMApp(appAttemptId, rm.GetRMContext());
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "default", "user");
fs.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false
);
fs.Handle(attemptEvent);
// Verify the blacklist can be updated independent of requesting containers
fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections
.EmptyList <ContainerId>(), Sharpen.Collections.SingletonList(host), null);
NUnit.Framework.Assert.IsTrue(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted
(host));
fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections
.EmptyList <ContainerId>(), null, Sharpen.Collections.SingletonList(host));
NUnit.Framework.Assert.IsFalse(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted
(host));
rm.Stop();
}
// To hold list of application for which event was received
/// <exception cref="System.Exception"/>
public virtual void TestNodeUsableEvent()
{
Logger rootLogger = LogManager.GetRootLogger();
rootLogger.SetLevel(Level.Debug);
Dispatcher dispatcher = GetDispatcher();
YarnConfiguration conf = new YarnConfiguration();
MockRM rm = new _MockRM_62(dispatcher, conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 28000);
NodesListManager nodesListManager = rm.GetNodesListManager();
Resource clusterResource = Resource.NewInstance(28000, 8);
RMNode rmnode = MockNodes.NewNodeInfo(1, clusterResource);
// Create killing APP
RMApp killrmApp = rm.SubmitApp(200);
rm.KillApp(killrmApp.GetApplicationId());
rm.WaitForState(killrmApp.GetApplicationId(), RMAppState.Killed);
// Create finish APP
RMApp finshrmApp = rm.SubmitApp(2000);
nm1.NodeHeartbeat(true);
RMAppAttempt attempt = finshrmApp.GetCurrentAppAttempt();
MockAM am = rm.SendAMLaunched(attempt.GetAppAttemptId());
am.RegisterAppAttempt();
am.UnregisterAppAttempt();
nm1.NodeHeartbeat(attempt.GetAppAttemptId(), 1, ContainerState.Complete);
am.WaitForState(RMAppAttemptState.Finished);
// Create submitted App
RMApp subrmApp = rm.SubmitApp(200);
// Fire Event for NODE_USABLE
nodesListManager.Handle(new NodesListManagerEvent(NodesListManagerEventType.NodeUsable
, rmnode));
if (applist.Count > 0)
{
NUnit.Framework.Assert.IsTrue("Event based on running app expected " + subrmApp.GetApplicationId
(), applist.Contains(subrmApp.GetApplicationId()));
NUnit.Framework.Assert.IsFalse("Event based on finish app not expected " + finshrmApp
.GetApplicationId(), applist.Contains(finshrmApp.GetApplicationId()));
NUnit.Framework.Assert.IsFalse("Event based on killed app not expected " + killrmApp
.GetApplicationId(), applist.Contains(killrmApp.GetApplicationId()));
}
else
{
NUnit.Framework.Assert.Fail("Events received should have beeen more than 1");
}
applist.Clear();
// Fire Event for NODE_UNUSABLE
nodesListManager.Handle(new NodesListManagerEvent(NodesListManagerEventType.NodeUnusable
, rmnode));
if (applist.Count > 0)
{
NUnit.Framework.Assert.IsTrue("Event based on running app expected " + subrmApp.GetApplicationId
(), applist.Contains(subrmApp.GetApplicationId()));
NUnit.Framework.Assert.IsFalse("Event based on finish app not expected " + finshrmApp
.GetApplicationId(), applist.Contains(finshrmApp.GetApplicationId()));
NUnit.Framework.Assert.IsFalse("Event based on killed app not expected " + killrmApp
.GetApplicationId(), applist.Contains(killrmApp.GetApplicationId()));
}
else
{
NUnit.Framework.Assert.Fail("Events received should have beeen more than 1");
}
}
/// <exception cref="System.Exception"/>
public virtual void TestIsStarvedForFairShare()
{
conf.Set(FairSchedulerConfiguration.AllocationFile, AllocFile);
PrintWriter @out = new PrintWriter(new FileWriter(AllocFile));
@out.WriteLine("<?xml version=\"1.0\"?>");
@out.WriteLine("<allocations>");
@out.WriteLine("<queue name=\"queueA\">");
@out.WriteLine("<weight>.2</weight>");
@out.WriteLine("</queue>");
@out.WriteLine("<queue name=\"queueB\">");
@out.WriteLine("<weight>.8</weight>");
@out.WriteLine("<fairSharePreemptionThreshold>.4</fairSharePreemptionThreshold>");
@out.WriteLine("<queue name=\"queueB1\">");
@out.WriteLine("</queue>");
@out.WriteLine("<queue name=\"queueB2\">");
@out.WriteLine("<fairSharePreemptionThreshold>.6</fairSharePreemptionThreshold>");
@out.WriteLine("</queue>");
@out.WriteLine("</queue>");
@out.WriteLine("<defaultFairSharePreemptionThreshold>.5</defaultFairSharePreemptionThreshold>"
);
@out.WriteLine("</allocations>");
@out.Close();
resourceManager = new MockRM(conf);
resourceManager.Start();
scheduler = (FairScheduler)resourceManager.GetResourceScheduler();
// Add one big node (only care about aggregate capacity)
RMNode node1 = MockNodes.NewNodeInfo(1, Resources.CreateResource(10 * 1024, 10),
1, "127.0.0.1");
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
scheduler.Handle(nodeEvent1);
scheduler.Update();
// Queue A wants 4 * 1024. Node update gives this all to A
CreateSchedulingRequest(1 * 1024, "queueA", "user1", 4);
scheduler.Update();
NodeUpdateSchedulerEvent nodeEvent2 = new NodeUpdateSchedulerEvent(node1);
for (int i = 0; i < 4; i++)
{
scheduler.Handle(nodeEvent2);
}
QueueManager queueMgr = scheduler.GetQueueManager();
FSLeafQueue queueA = queueMgr.GetLeafQueue("queueA", false);
NUnit.Framework.Assert.AreEqual(4 * 1024, queueA.GetResourceUsage().GetMemory());
// Both queue B1 and queue B2 want 3 * 1024
CreateSchedulingRequest(1 * 1024, "queueB.queueB1", "user1", 3);
CreateSchedulingRequest(1 * 1024, "queueB.queueB2", "user1", 3);
scheduler.Update();
for (int i_1 = 0; i_1 < 4; i_1++)
{
scheduler.Handle(nodeEvent2);
}
FSLeafQueue queueB1 = queueMgr.GetLeafQueue("queueB.queueB1", false);
FSLeafQueue queueB2 = queueMgr.GetLeafQueue("queueB.queueB2", false);
NUnit.Framework.Assert.AreEqual(2 * 1024, queueB1.GetResourceUsage().GetMemory());
NUnit.Framework.Assert.AreEqual(2 * 1024, queueB2.GetResourceUsage().GetMemory());
// For queue B1, the fairSharePreemptionThreshold is 0.4, and the fair share
// threshold is 1.6 * 1024
NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare());
// For queue B2, the fairSharePreemptionThreshold is 0.6, and the fair share
// threshold is 2.4 * 1024
NUnit.Framework.Assert.IsTrue(queueB2.IsStarvedForFairShare());
// Node checks in again
scheduler.Handle(nodeEvent2);
scheduler.Handle(nodeEvent2);
NUnit.Framework.Assert.AreEqual(3 * 1024, queueB1.GetResourceUsage().GetMemory());
NUnit.Framework.Assert.AreEqual(3 * 1024, queueB2.GetResourceUsage().GetMemory());
// Both queue B1 and queue B2 usages go to 3 * 1024
NUnit.Framework.Assert.IsFalse(queueB1.IsStarvedForFairShare());
NUnit.Framework.Assert.IsFalse(queueB2.IsStarvedForFairShare());
}
/// <exception cref="System.Exception"/>
public virtual void TestUpdateResourceOnNode()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.RollMasterKey();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
FifoScheduler scheduler = new _FifoScheduler_275(this);
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration());
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(2048, 4), 1, "127.0.0.1"
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
scheduler.Handle(nodeEvent1);
MethodInfo method = Sharpen.Runtime.GetDeclaredMethod(scheduler.GetType(), "getNodes"
);
IDictionary <NodeId, FiCaSchedulerNode> schedulerNodes = (IDictionary <NodeId, FiCaSchedulerNode
>)method.Invoke(scheduler);
NUnit.Framework.Assert.AreEqual(schedulerNodes.Values.Count, 1);
Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = Resources.CreateResource
(1024, 4);
NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent
(node0, ResourceOption.NewInstance(newResource, RMNode.OverCommitTimeoutMillisDefault
));
scheduler.Handle(node0ResourceUpdate);
// SchedulerNode's total resource and available resource are changed.
NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetTotalResource
().GetMemory(), 1024);
NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetAvailableResource
().GetMemory(), 1024);
QueueInfo queueInfo = scheduler.GetQueueInfo(null, false, false);
NUnit.Framework.Assert.AreEqual(0.0f, queueInfo.GetCurrentCapacity(), 0.0f);
int _appId = 1;
int _appAttemptId = 1;
ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId);
CreateMockRMApp(appAttemptId, rmContext);
AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId
(), "queue1", "user1");
scheduler.Handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId
, false);
scheduler.Handle(attemptEvent);
int memory = 1024;
int priority = 1;
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority
, 1);
ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority
, 1);
ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority
, 1);
ask.AddItem(nodeLocal);
ask.AddItem(rackLocal);
ask.AddItem(any);
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
// Before the node update event, there are one local request
NUnit.Framework.Assert.AreEqual(1, nodeLocal.GetNumContainers());
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
// Now schedule.
scheduler.Handle(node0Update);
// After the node update event, check no local request
NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers());
// Also check that one container was scheduled
SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId);
NUnit.Framework.Assert.AreEqual(1, info.GetLiveContainers().Count);
// And check the default Queue now is full.
queueInfo = scheduler.GetQueueInfo(null, false, false);
NUnit.Framework.Assert.AreEqual(1.0f, queueInfo.GetCurrentCapacity(), 0.0f);
}
/// <exception cref="System.Exception"/>
public virtual void TestNodeLocalAssignment()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.RollMasterKey();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
FifoScheduler scheduler = new FifoScheduler();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration());
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(1024 * 64), 1, "127.0.0.1"
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
scheduler.Handle(nodeEvent1);
int _appId = 1;
int _appAttemptId = 1;
ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId);
CreateMockRMApp(appAttemptId, rmContext);
AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId
(), "queue1", "user1");
scheduler.Handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId
, false);
scheduler.Handle(attemptEvent);
int memory = 64;
int nConts = 3;
int priority = 20;
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority
, nConts);
ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority
, nConts);
ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority
, nConts);
ask.AddItem(nodeLocal);
ask.AddItem(rackLocal);
ask.AddItem(any);
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
// Before the node update event, there are 3 local requests outstanding
NUnit.Framework.Assert.AreEqual(3, nodeLocal.GetNumContainers());
scheduler.Handle(node0Update);
// After the node update event, check that there are no more local requests
// outstanding
NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers());
//Also check that the containers were scheduled
SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId);
NUnit.Framework.Assert.AreEqual(3, info.GetLiveContainers().Count);
scheduler.Stop();
}