/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public virtual void Submit()
{
lock (this)
{
ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext
>();
context.SetApplicationId(this.applicationId);
context.SetQueue(this.queue);
// Set up the container launch context for the application master
ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord
<ContainerLaunchContext>();
context.SetAMContainerSpec(amContainer);
context.SetResource(Resources.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb
));
SubmitApplicationRequest request = recordFactory.NewRecordInstance <SubmitApplicationRequest
>();
request.SetApplicationSubmissionContext(context);
ResourceScheduler scheduler = resourceManager.GetResourceScheduler();
resourceManager.GetClientRMService().SubmitApplication(request);
// Notify scheduler
AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId
, this.queue, "user");
scheduler.Handle(addAppEvent);
AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent
(this.applicationAttemptId, false);
scheduler.Handle(addAttemptEvent);
}
}
protected internal virtual void CreateApplicationWithAMResource(ApplicationAttemptId
attId, string queue, string user, Resource amResource)
{
RMContext rmContext = resourceManager.GetRMContext();
RMApp rmApp = new RMAppImpl(attId.GetApplicationId(), rmContext, conf, null, null
, null, ApplicationSubmissionContext.NewInstance(null, null, null, null, null, false
, false, 0, amResource, null), null, null, 0, null, null, null);
rmContext.GetRMApps()[attId.GetApplicationId()] = rmApp;
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attId.GetApplicationId
(), queue, user);
scheduler.Handle(appAddedEvent);
AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent
(attId, false);
scheduler.Handle(attempAddedEvent);
}
/// <exception cref="System.Exception"/>
public static SchedulerApplication <SchedulerApplicationAttempt> VerifyAppAddedAndRemovedFromScheduler
(IDictionary <ApplicationId, SchedulerApplication <SchedulerApplicationAttempt> > applications
, EventHandler <SchedulerEvent> handler, string queueName)
{
ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1);
AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appId, queueName
, "user");
handler.Handle(appAddedEvent);
SchedulerApplication <SchedulerApplicationAttempt> app = applications[appId];
// verify application is added.
NUnit.Framework.Assert.IsNotNull(app);
NUnit.Framework.Assert.AreEqual("user", app.GetUser());
AppRemovedSchedulerEvent appRemoveEvent = new AppRemovedSchedulerEvent(appId, RMAppState
.Finished);
handler.Handle(appRemoveEvent);
NUnit.Framework.Assert.IsNull(applications[appId]);
return(app);
}
/// <exception cref="System.Exception"/>
public virtual void TestAppAttemptMetrics()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
FifoScheduler scheduler = new FifoScheduler();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
null, null, null, scheduler);
((RMContextImpl)rmContext).SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
Configuration conf = new Configuration();
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(conf, rmContext);
QueueMetrics metrics = scheduler.GetRootQueueMetrics();
int beforeAppsSubmitted = metrics.GetAppsSubmitted();
ApplicationId appId = BuilderUtils.NewApplicationId(200, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");
scheduler.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false
);
scheduler.Handle(attemptEvent);
appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 2);
SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false
);
scheduler.Handle(attemptEvent2);
int afterAppsSubmitted = metrics.GetAppsSubmitted();
NUnit.Framework.Assert.AreEqual(1, afterAppsSubmitted - beforeAppsSubmitted);
scheduler.Stop();
}
public virtual void TestBlackListNodes()
{
Configuration conf = new Configuration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
MockRM rm = new MockRM(conf);
rm.Start();
FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
string host = "127.0.0.1";
RMNode node = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, host);
fs.Handle(new NodeAddedSchedulerEvent(node));
ApplicationId appId = BuilderUtils.NewApplicationId(100, 1);
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
);
CreateMockRMApp(appAttemptId, rm.GetRMContext());
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "default", "user");
fs.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false
);
fs.Handle(attemptEvent);
// Verify the blacklist can be updated independent of requesting containers
fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections
.EmptyList <ContainerId>(), Sharpen.Collections.SingletonList(host), null);
NUnit.Framework.Assert.IsTrue(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted
(host));
fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections
.EmptyList <ContainerId>(), null, Sharpen.Collections.SingletonList(host));
NUnit.Framework.Assert.IsFalse(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted
(host));
rm.Stop();
}
public override void Handle(SchedulerEvent @event)
{
switch (@event.GetType())
{
case SchedulerEventType.NodeAdded:
{
NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)@event;
AddNode(nodeAddedEvent.GetAddedRMNode());
RecoverContainersOnNode(nodeAddedEvent.GetContainerReports(), nodeAddedEvent.GetAddedRMNode
());
break;
}
case SchedulerEventType.NodeRemoved:
{
NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)@event;
RemoveNode(nodeRemovedEvent.GetRemovedRMNode());
break;
}
case SchedulerEventType.NodeResourceUpdate:
{
NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = (NodeResourceUpdateSchedulerEvent
)@event;
UpdateNodeResource(nodeResourceUpdatedEvent.GetRMNode(), nodeResourceUpdatedEvent
.GetResourceOption());
break;
}
case SchedulerEventType.NodeUpdate:
{
NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)@event;
NodeUpdate(nodeUpdatedEvent.GetRMNode());
break;
}
case SchedulerEventType.AppAdded:
{
AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)@event;
AddApplication(appAddedEvent.GetApplicationId(), appAddedEvent.GetQueue(), appAddedEvent
.GetUser(), appAddedEvent.GetIsAppRecovering());
break;
}
case SchedulerEventType.AppRemoved:
{
AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)@event;
DoneApplication(appRemovedEvent.GetApplicationID(), appRemovedEvent.GetFinalState
());
break;
}
case SchedulerEventType.AppAttemptAdded:
{
AppAttemptAddedSchedulerEvent appAttemptAddedEvent = (AppAttemptAddedSchedulerEvent
)@event;
AddApplicationAttempt(appAttemptAddedEvent.GetApplicationAttemptId(), appAttemptAddedEvent
.GetTransferStateFromPreviousAttempt(), appAttemptAddedEvent.GetIsAttemptRecovering
());
break;
}
case SchedulerEventType.AppAttemptRemoved:
{
AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = (AppAttemptRemovedSchedulerEvent
)@event;
try
{
DoneApplicationAttempt(appAttemptRemovedEvent.GetApplicationAttemptID(), appAttemptRemovedEvent
.GetFinalAttemptState(), appAttemptRemovedEvent.GetKeepContainersAcrossAppAttempts
());
}
catch (IOException ie)
{
Log.Error("Unable to remove application " + appAttemptRemovedEvent.GetApplicationAttemptID
(), ie);
}
break;
}
case SchedulerEventType.ContainerExpired:
{
ContainerExpiredSchedulerEvent containerExpiredEvent = (ContainerExpiredSchedulerEvent
)@event;
ContainerId containerid = containerExpiredEvent.GetContainerId();
CompletedContainer(GetRMContainer(containerid), SchedulerUtils.CreateAbnormalContainerStatus
(containerid, SchedulerUtils.ExpiredContainer), RMContainerEventType.Expire);
break;
}
case SchedulerEventType.ContainerRescheduled:
{
ContainerRescheduledEvent containerRescheduledEvent = (ContainerRescheduledEvent)
@event;
RMContainer container = containerRescheduledEvent.GetContainer();
RecoverResourceRequestForContainer(container);
break;
}
default:
{
Log.Error("Invalid eventtype " + @event.GetType() + ". Ignoring!");
break;
}
}
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
/// <exception cref="System.Exception"/>
/// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/>
protected internal virtual void TestPlanFollower(bool isMove)
{
// Initialize plan based on move flag
plan = new InMemoryPlan(scheduler.GetRootQueueMetrics(), policy, mAgent, scheduler
.GetClusterResource(), 1L, res, scheduler.GetMinimumResourceCapability(), maxAlloc
, "dedicated", null, isMove);
// add a few reservations to the plan
long ts = Runtime.CurrentTimeMillis();
ReservationId r1 = ReservationId.NewInstance(ts, 1);
int[] f1 = new int[] { 10, 10, 10, 10, 10 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r1, null, "u3", "dedicated", 0, 0 + f1.Length, ReservationSystemTestUtil.GenerateAllocation
(0L, 1L, f1), res, minAlloc)));
ReservationId r2 = ReservationId.NewInstance(ts, 2);
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r2, null, "u3", "dedicated", 3, 3 + f1.Length, ReservationSystemTestUtil.GenerateAllocation
(3L, 1L, f1), res, minAlloc)));
ReservationId r3 = ReservationId.NewInstance(ts, 3);
int[] f2 = new int[] { 0, 10, 20, 10, 0 };
NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation
(r3, null, "u4", "dedicated", 10, 10 + f2.Length, ReservationSystemTestUtil.GenerateAllocation
(10L, 1L, f2), res, minAlloc)));
AbstractSchedulerPlanFollower planFollower = CreatePlanFollower();
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(0L);
planFollower.Run();
Queue q = GetReservationQueue(r1.ToString());
AssertReservationQueueExists(r1);
// submit an app to r1
string user_0 = "test-user";
ApplicationId appId = ApplicationId.NewInstance(0, 1);
ApplicationAttemptId appAttemptId_0 = ApplicationAttemptId.NewInstance(appId, 0);
AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, q.GetQueueName
(), user_0);
scheduler.Handle(addAppEvent);
AppAttemptAddedSchedulerEvent appAttemptAddedEvent = new AppAttemptAddedSchedulerEvent
(appAttemptId_0, false);
scheduler.Handle(appAttemptAddedEvent);
// initial default reservation queue should have no apps
Queue defQ = GetDefaultQueue();
NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ));
AssertReservationQueueExists(r1, 0.1, 0.1);
NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(q));
AssertReservationQueueDoesNotExist(r2);
AssertReservationQueueDoesNotExist(r3);
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(3L);
planFollower.Run();
NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ));
AssertReservationQueueExists(r1, 0.1, 0.1);
NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(q));
AssertReservationQueueExists(r2, 0.1, 0.1);
AssertReservationQueueDoesNotExist(r3);
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(10L);
planFollower.Run();
q = GetReservationQueue(r1.ToString());
if (isMove)
{
// app should have been moved to default reservation queue
NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(defQ));
NUnit.Framework.Assert.IsNull(q);
}
else
{
// app should be killed
NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ));
NUnit.Framework.Assert.IsNotNull(q);
AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = new AppAttemptRemovedSchedulerEvent
(appAttemptId_0, RMAppAttemptState.Killed, false);
scheduler.Handle(appAttemptRemovedEvent);
}
AssertReservationQueueDoesNotExist(r2);
AssertReservationQueueExists(r3, 0, 1.0);
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(11L);
planFollower.Run();
if (isMove)
{
// app should have been moved to default reservation queue
NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(defQ));
}
else
{
// app should be killed
NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ));
}
AssertReservationQueueDoesNotExist(r1);
AssertReservationQueueDoesNotExist(r2);
AssertReservationQueueExists(r3, 0.1, 0.1);
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(12L);
planFollower.Run();
AssertReservationQueueDoesNotExist(r1);
AssertReservationQueueDoesNotExist(r2);
AssertReservationQueueExists(r3, 0.2, 0.2);
Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(16L);
planFollower.Run();
AssertReservationQueueDoesNotExist(r1);
AssertReservationQueueDoesNotExist(r2);
AssertReservationQueueDoesNotExist(r3);
VerifyCapacity(defQ);
}
/// <exception cref="System.Exception"/>
public virtual void TestHeadroom()
{
Configuration conf = new Configuration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
MockRM rm = new MockRM(conf);
rm.Start();
FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
// Add a node
RMNode n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, "127.0.0.2"
);
fs.Handle(new NodeAddedSchedulerEvent(n1));
// Add two applications
ApplicationId appId1 = BuilderUtils.NewApplicationId(100, 1);
ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1,
1);
CreateMockRMApp(appAttemptId1, rm.GetRMContext());
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
fs.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false
);
fs.Handle(attemptEvent);
ApplicationId appId2 = BuilderUtils.NewApplicationId(200, 2);
ApplicationAttemptId appAttemptId2 = BuilderUtils.NewApplicationAttemptId(appId2,
1);
CreateMockRMApp(appAttemptId2, rm.GetRMContext());
SchedulerEvent appEvent2 = new AppAddedSchedulerEvent(appId2, "queue", "user");
fs.Handle(appEvent2);
SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false
);
fs.Handle(attemptEvent2);
IList <ContainerId> emptyId = new AList <ContainerId>();
IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>();
// Set up resource requests
// Ask for a 1 GB container for app 1
IList <ResourceRequest> ask1 = new AList <ResourceRequest>();
ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(Gb, 1), 1));
fs.Allocate(appAttemptId1, ask1, emptyId, null, null);
// Ask for a 2 GB container for app 2
IList <ResourceRequest> ask2 = new AList <ResourceRequest>();
ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(2 * Gb, 1), 1));
fs.Allocate(appAttemptId2, ask2, emptyId, null, null);
// Trigger container assignment
fs.Handle(new NodeUpdateSchedulerEvent(n1));
// Get the allocation for the applications and verify headroom
Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation1.GetResourceLimit
().GetMemory());
Allocation allocation2 = fs.Allocate(appAttemptId2, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation2.GetResourceLimit
().GetMemory());
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestBlackListNodes()
{
Configuration conf = new Configuration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
));
MockRM rm = new MockRM(conf);
rm.Start();
FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
int rack_num_0 = 0;
int rack_num_1 = 1;
// Add 4 nodes in 2 racks
// host_0_0 in rack0
string host_0_0 = "127.0.0.1";
RMNode n1 = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_0
);
fs.Handle(new NodeAddedSchedulerEvent(n1));
// host_0_1 in rack0
string host_0_1 = "127.0.0.2";
RMNode n2 = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_1
);
fs.Handle(new NodeAddedSchedulerEvent(n2));
// host_1_0 in rack1
string host_1_0 = "127.0.0.3";
RMNode n3 = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_0
);
fs.Handle(new NodeAddedSchedulerEvent(n3));
// host_1_1 in rack1
string host_1_1 = "127.0.0.4";
RMNode n4 = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_1
);
fs.Handle(new NodeAddedSchedulerEvent(n4));
// Add one application
ApplicationId appId1 = BuilderUtils.NewApplicationId(100, 1);
ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1,
1);
CreateMockRMApp(appAttemptId1, rm.GetRMContext());
SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");
fs.Handle(appEvent);
SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false
);
fs.Handle(attemptEvent);
IList <ContainerId> emptyId = new AList <ContainerId>();
IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>();
// Allow rack-locality for rack_1, but blacklist host_1_0
// Set up resource requests
// Ask for a 1 GB container for app 1
IList <ResourceRequest> ask1 = new AList <ResourceRequest>();
ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), "rack1"
, BuilderUtils.NewResource(Gb, 1), 1));
ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(Gb, 1), 1));
fs.Allocate(appAttemptId1, ask1, emptyId, Sharpen.Collections.SingletonList(host_1_0
), null);
// Trigger container assignment
fs.Handle(new NodeUpdateSchedulerEvent(n3));
// Get the allocation for the application and verify no allocation on blacklist node
Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation1", 0, allocation1.GetContainers().Count
);
// verify host_1_1 can get allocated as not in blacklist
fs.Handle(new NodeUpdateSchedulerEvent(n4));
Allocation allocation2 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation2", 1, allocation2.GetContainers().Count
);
IList <Container> containerList = allocation2.GetContainers();
foreach (Container container in containerList)
{
NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container.GetNodeId
(), n4.GetNodeID());
}
// Ask for a 1 GB container again for app 1
IList <ResourceRequest> ask2 = new AList <ResourceRequest>();
// this time, rack0 is also in blacklist, so only host_1_1 is available to
// be assigned
ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
.Any, BuilderUtils.NewResource(Gb, 1), 1));
fs.Allocate(appAttemptId1, ask2, emptyId, Sharpen.Collections.SingletonList("rack0"
), null);
// verify n1 is not qualified to be allocated
fs.Handle(new NodeUpdateSchedulerEvent(n1));
Allocation allocation3 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation3", 0, allocation3.GetContainers().Count
);
// verify n2 is not qualified to be allocated
fs.Handle(new NodeUpdateSchedulerEvent(n2));
Allocation allocation4 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation4", 0, allocation4.GetContainers().Count
);
// verify n3 is not qualified to be allocated
fs.Handle(new NodeUpdateSchedulerEvent(n3));
Allocation allocation5 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation5", 0, allocation5.GetContainers().Count
);
fs.Handle(new NodeUpdateSchedulerEvent(n4));
Allocation allocation6 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
);
NUnit.Framework.Assert.AreEqual("allocation6", 1, allocation6.GetContainers().Count
);
containerList = allocation6.GetContainers();
foreach (Container container_1 in containerList)
{
NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container_1.GetNodeId
(), n4.GetNodeID());
}
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestUpdateResourceOnNode()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.RollMasterKey();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
FifoScheduler scheduler = new _FifoScheduler_275(this);
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration());
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(2048, 4), 1, "127.0.0.1"
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
scheduler.Handle(nodeEvent1);
MethodInfo method = Sharpen.Runtime.GetDeclaredMethod(scheduler.GetType(), "getNodes"
);
IDictionary <NodeId, FiCaSchedulerNode> schedulerNodes = (IDictionary <NodeId, FiCaSchedulerNode
>)method.Invoke(scheduler);
NUnit.Framework.Assert.AreEqual(schedulerNodes.Values.Count, 1);
Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = Resources.CreateResource
(1024, 4);
NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent
(node0, ResourceOption.NewInstance(newResource, RMNode.OverCommitTimeoutMillisDefault
));
scheduler.Handle(node0ResourceUpdate);
// SchedulerNode's total resource and available resource are changed.
NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetTotalResource
().GetMemory(), 1024);
NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetAvailableResource
().GetMemory(), 1024);
QueueInfo queueInfo = scheduler.GetQueueInfo(null, false, false);
NUnit.Framework.Assert.AreEqual(0.0f, queueInfo.GetCurrentCapacity(), 0.0f);
int _appId = 1;
int _appAttemptId = 1;
ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId);
CreateMockRMApp(appAttemptId, rmContext);
AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId
(), "queue1", "user1");
scheduler.Handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId
, false);
scheduler.Handle(attemptEvent);
int memory = 1024;
int priority = 1;
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority
, 1);
ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority
, 1);
ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority
, 1);
ask.AddItem(nodeLocal);
ask.AddItem(rackLocal);
ask.AddItem(any);
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
// Before the node update event, there are one local request
NUnit.Framework.Assert.AreEqual(1, nodeLocal.GetNumContainers());
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
// Now schedule.
scheduler.Handle(node0Update);
// After the node update event, check no local request
NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers());
// Also check that one container was scheduled
SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId);
NUnit.Framework.Assert.AreEqual(1, info.GetLiveContainers().Count);
// And check the default Queue now is full.
queueInfo = scheduler.GetQueueInfo(null, false, false);
NUnit.Framework.Assert.AreEqual(1.0f, queueInfo.GetCurrentCapacity(), 0.0f);
}
/// <exception cref="System.Exception"/>
public virtual void TestNodeLocalAssignment()
{
AsyncDispatcher dispatcher = new InlineDispatcher();
Configuration conf = new Configuration();
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf
);
nmTokenSecretManager.RollMasterKey();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
FifoScheduler scheduler = new FifoScheduler();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>());
rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>());
((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration());
scheduler.SetRMContext(rmContext);
scheduler.Init(conf);
scheduler.Start();
scheduler.Reinitialize(new Configuration(), rmContext);
RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(1024 * 64), 1, "127.0.0.1"
);
NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0);
scheduler.Handle(nodeEvent1);
int _appId = 1;
int _appAttemptId = 1;
ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId);
CreateMockRMApp(appAttemptId, rmContext);
AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId
(), "queue1", "user1");
scheduler.Handle(appEvent);
AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId
, false);
scheduler.Handle(attemptEvent);
int memory = 64;
int nConts = 3;
int priority = 20;
IList <ResourceRequest> ask = new AList <ResourceRequest>();
ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority
, nConts);
ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority
, nConts);
ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority
, nConts);
ask.AddItem(nodeLocal);
ask.AddItem(rackLocal);
ask.AddItem(any);
scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null);
NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0);
// Before the node update event, there are 3 local requests outstanding
NUnit.Framework.Assert.AreEqual(3, nodeLocal.GetNumContainers());
scheduler.Handle(node0Update);
// After the node update event, check that there are no more local requests
// outstanding
NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers());
//Also check that the containers were scheduled
SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId);
NUnit.Framework.Assert.AreEqual(3, info.GetLiveContainers().Count);
scheduler.Stop();
}
