public virtual void TestNodeUpdateBeforeAppAttemptInit()
{
FifoScheduler scheduler = new FifoScheduler();
MockRM rm = new MockRM(conf);
scheduler.SetRMContext(rm.GetRMContext());
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(conf, rm.GetRMContext());
RMNode node = MockNodes.NewNodeInfo(1, Resources.CreateResource(1024, 4), 1, "127.0.0.1"
);
scheduler.Handle(new NodeAddedSchedulerEvent(node));
ApplicationId appId = ApplicationId.NewInstance(0, 1);
scheduler.AddApplication(appId, "queue1", "user1", false);
NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
try
{
scheduler.Handle(updateEvent);
}
catch (ArgumentNullException)
{
NUnit.Framework.Assert.Fail();
}
ApplicationAttemptId attId = ApplicationAttemptId.NewInstance(appId, 1);
scheduler.AddApplicationAttempt(attId, false, false);
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestReconnectedNode()
{
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
conf.SetQueues("default", new string[] { "default" });
conf.SetCapacity("default", 100);
FifoScheduler fs = new FifoScheduler();
fs.Init(conf);
fs.Start();
// mock rmContext to avoid NPE.
RMContext context = Org.Mockito.Mockito.Mock <RMContext>();
fs.Reinitialize(conf, null);
fs.SetRMContext(context);
RMNode n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, "127.0.0.2"
);
RMNode n2 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(2 * Gb), 2, "127.0.0.3"
);
fs.Handle(new NodeAddedSchedulerEvent(n1));
fs.Handle(new NodeAddedSchedulerEvent(n2));
fs.Handle(new NodeUpdateSchedulerEvent(n1));
NUnit.Framework.Assert.AreEqual(6 * Gb, fs.GetRootQueueMetrics().GetAvailableMB()
);
// reconnect n1 with downgraded memory
n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(2 * Gb), 1, "127.0.0.2");
fs.Handle(new NodeRemovedSchedulerEvent(n1));
fs.Handle(new NodeAddedSchedulerEvent(n1));
fs.Handle(new NodeUpdateSchedulerEvent(n1));
NUnit.Framework.Assert.AreEqual(4 * Gb, fs.GetRootQueueMetrics().GetAvailableMB()
);
fs.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestHeadroom()
{
Configuration conf = new Configuration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
MockRM rm = new MockRM(conf);
rm.Start();
FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
// Add a node
RMNode n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, "127.0.0.2"
);
fs.Handle(new NodeAddedSchedulerEvent(n1));
// Add two applications
ApplicationId appId1 = BuilderUtils.NewApplicationId(100, 1);
ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1,
1);
CreateMockRMApp(appAttemptId1, rm.GetRMContext());
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
fs.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false
);
fs.Handle(attemptEvent);
ApplicationId appId2 = BuilderUtils.NewApplicationId(200, 2);
ApplicationAttemptId appAttemptId2 = BuilderUtils.NewApplicationAttemptId(appId2,
1);
CreateMockRMApp(appAttemptId2, rm.GetRMContext());
SchedulerEvent appEvent2 = new AppAddedSchedulerEvent(appId2, "queue", "user");
fs.Handle(appEvent2);
SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false
);
fs.Handle(attemptEvent2);
IList <ContainerId> emptyId = new AList <ContainerId>();
IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>();
// Set up resource requests
// Ask for a 1 GB container for app 1
IList <ResourceRequest> ask1 = new AList <ResourceRequest>();
ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(Gb, 1), 1));
fs.Allocate(appAttemptId1, ask1, emptyId, null, null);
// Ask for a 2 GB container for app 2
IList <ResourceRequest> ask2 = new AList <ResourceRequest>();
ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(2 * Gb, 1), 1));
fs.Allocate(appAttemptId2, ask2, emptyId, null, null);
// Trigger container assignment
fs.Handle(new NodeUpdateSchedulerEvent(n1));
// Get the allocation for the applications and verify headroom
Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation1.GetResourceLimit
().GetMemory());
Allocation allocation2 = fs.Allocate(appAttemptId2, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation2.GetResourceLimit
().GetMemory());
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestBlackListNodes()
{
Configuration conf = new Configuration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
MockRM rm = new MockRM(conf);
rm.Start();
FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
int rack_num_0 = 0;
int rack_num_1 = 1;
// Add 4 nodes in 2 racks
// host_0_0 in rack0
string host_0_0 = "127.0.0.1";
RMNode n1 = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_0
);
fs.Handle(new NodeAddedSchedulerEvent(n1));
// host_0_1 in rack0
string host_0_1 = "127.0.0.2";
RMNode n2 = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_1
);
fs.Handle(new NodeAddedSchedulerEvent(n2));
// host_1_0 in rack1
string host_1_0 = "127.0.0.3";
RMNode n3 = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_0
);
fs.Handle(new NodeAddedSchedulerEvent(n3));
// host_1_1 in rack1
string host_1_1 = "127.0.0.4";
RMNode n4 = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_1
);
fs.Handle(new NodeAddedSchedulerEvent(n4));
// Add one application
ApplicationId appId1 = BuilderUtils.NewApplicationId(100, 1);
ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1,
1);
CreateMockRMApp(appAttemptId1, rm.GetRMContext());
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
fs.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false
);
fs.Handle(attemptEvent);
IList <ContainerId> emptyId = new AList <ContainerId>();
IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>();
// Allow rack-locality for rack_1, but blacklist host_1_0
// Set up resource requests
// Ask for a 1 GB container for app 1
IList <ResourceRequest> ask1 = new AList <ResourceRequest>();
ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), "rack1"
, BuilderUtils.NewResource(Gb, 1), 1));
ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(Gb, 1), 1));
fs.Allocate(appAttemptId1, ask1, emptyId, Sharpen.Collections.SingletonList(host_1_0
), null);
// Trigger container assignment
fs.Handle(new NodeUpdateSchedulerEvent(n3));
// Get the allocation for the application and verify no allocation on blacklist node
Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation1", 0, allocation1.GetContainers().Count
);
// verify host_1_1 can get allocated as not in blacklist
fs.Handle(new NodeUpdateSchedulerEvent(n4));
Allocation allocation2 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation2", 1, allocation2.GetContainers().Count
);
IList <Container> containerList = allocation2.GetContainers();
foreach (Container container in containerList)
{
NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container.GetNodeId
(), n4.GetNodeID());
}
// Ask for a 1 GB container again for app 1
IList <ResourceRequest> ask2 = new AList <ResourceRequest>();
// this time, rack0 is also in blacklist, so only host_1_1 is available to
// be assigned
ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(Gb, 1), 1));
fs.Allocate(appAttemptId1, ask2, emptyId, Sharpen.Collections.SingletonList("rack0"
), null);
// verify n1 is not qualified to be allocated
fs.Handle(new NodeUpdateSchedulerEvent(n1));
Allocation allocation3 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation3", 0, allocation3.GetContainers().Count
);
// verify n2 is not qualified to be allocated
fs.Handle(new NodeUpdateSchedulerEvent(n2));
Allocation allocation4 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation4", 0, allocation4.GetContainers().Count
);
// verify n3 is not qualified to be allocated
fs.Handle(new NodeUpdateSchedulerEvent(n3));
Allocation allocation5 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation5", 0, allocation5.GetContainers().Count
);
fs.Handle(new NodeUpdateSchedulerEvent(n4));
Allocation allocation6 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation6", 1, allocation6.GetContainers().Count
);
containerList = allocation6.GetContainers();
foreach (Container container_1 in containerList)
{
NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container_1.GetNodeId
(), n4.GetNodeID());
}
rm.Stop();
}