/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void Submit() { lock (this) { ApplicationSubmissionContext context = recordFactory.NewRecordInstance <ApplicationSubmissionContext >(); context.SetApplicationId(this.applicationId); context.SetQueue(this.queue); // Set up the container launch context for the application master ContainerLaunchContext amContainer = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <ContainerLaunchContext>(); context.SetAMContainerSpec(amContainer); context.SetResource(Resources.CreateResource(YarnConfiguration.DefaultRmSchedulerMinimumAllocationMb )); SubmitApplicationRequest request = recordFactory.NewRecordInstance <SubmitApplicationRequest >(); request.SetApplicationSubmissionContext(context); ResourceScheduler scheduler = resourceManager.GetResourceScheduler(); resourceManager.GetClientRMService().SubmitApplication(request); // Notify scheduler AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(this.applicationId , this.queue, "user"); scheduler.Handle(addAppEvent); AppAttemptAddedSchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent (this.applicationAttemptId, false); scheduler.Handle(addAttemptEvent); } }
protected internal virtual void CreateApplicationWithAMResource(ApplicationAttemptId attId, string queue, string user, Resource amResource) { RMContext rmContext = resourceManager.GetRMContext(); RMApp rmApp = new RMAppImpl(attId.GetApplicationId(), rmContext, conf, null, null , null, ApplicationSubmissionContext.NewInstance(null, null, null, null, null, false , false, 0, amResource, null), null, null, 0, null, null, null); rmContext.GetRMApps()[attId.GetApplicationId()] = rmApp; AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(attId.GetApplicationId (), queue, user); scheduler.Handle(appAddedEvent); AppAttemptAddedSchedulerEvent attempAddedEvent = new AppAttemptAddedSchedulerEvent (attId, false); scheduler.Handle(attempAddedEvent); }
/// <exception cref="System.Exception"/> public static SchedulerApplication <SchedulerApplicationAttempt> VerifyAppAddedAndRemovedFromScheduler (IDictionary <ApplicationId, SchedulerApplication <SchedulerApplicationAttempt> > applications , EventHandler <SchedulerEvent> handler, string queueName) { ApplicationId appId = ApplicationId.NewInstance(Runtime.CurrentTimeMillis(), 1); AppAddedSchedulerEvent appAddedEvent = new AppAddedSchedulerEvent(appId, queueName , "user"); handler.Handle(appAddedEvent); SchedulerApplication <SchedulerApplicationAttempt> app = applications[appId]; // verify application is added. NUnit.Framework.Assert.IsNotNull(app); NUnit.Framework.Assert.AreEqual("user", app.GetUser()); AppRemovedSchedulerEvent appRemoveEvent = new AppRemovedSchedulerEvent(appId, RMAppState .Finished); handler.Handle(appRemoveEvent); NUnit.Framework.Assert.IsNull(applications[appId]); return(app); }
/// <exception cref="System.Exception"/> public virtual void TestAppAttemptMetrics() { AsyncDispatcher dispatcher = new InlineDispatcher(); FifoScheduler scheduler = new FifoScheduler(); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, scheduler); ((RMContextImpl)rmContext).SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher >()); Configuration conf = new Configuration(); scheduler.SetRMContext(rmContext); scheduler.Init(conf); scheduler.Start(); scheduler.Reinitialize(conf, rmContext); QueueMetrics metrics = scheduler.GetRootQueueMetrics(); int beforeAppsSubmitted = metrics.GetAppsSubmitted(); ApplicationId appId = BuilderUtils.NewApplicationId(200, 1); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user"); scheduler.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false ); scheduler.Handle(attemptEvent); appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 2); SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false ); scheduler.Handle(attemptEvent2); int afterAppsSubmitted = metrics.GetAppsSubmitted(); NUnit.Framework.Assert.AreEqual(1, afterAppsSubmitted - beforeAppsSubmitted); scheduler.Stop(); }
public virtual void TestBlackListNodes() { Configuration conf = new Configuration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); rm.Start(); FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler(); string host = "127.0.0.1"; RMNode node = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, host); fs.Handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = BuilderUtils.NewApplicationId(100, 1); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); CreateMockRMApp(appAttemptId, rm.GetRMContext()); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "default", "user"); fs.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false ); fs.Handle(attemptEvent); // Verify the blacklist can be updated independent of requesting containers fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections .EmptyList <ContainerId>(), Sharpen.Collections.SingletonList(host), null); NUnit.Framework.Assert.IsTrue(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted (host)); fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections .EmptyList <ContainerId>(), null, Sharpen.Collections.SingletonList(host)); NUnit.Framework.Assert.IsFalse(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted (host)); rm.Stop(); }
public override void Handle(SchedulerEvent @event) { switch (@event.GetType()) { case SchedulerEventType.NodeAdded: { NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent)@event; AddNode(nodeAddedEvent.GetAddedRMNode()); RecoverContainersOnNode(nodeAddedEvent.GetContainerReports(), nodeAddedEvent.GetAddedRMNode ()); break; } case SchedulerEventType.NodeRemoved: { NodeRemovedSchedulerEvent nodeRemovedEvent = (NodeRemovedSchedulerEvent)@event; RemoveNode(nodeRemovedEvent.GetRemovedRMNode()); break; } case SchedulerEventType.NodeResourceUpdate: { NodeResourceUpdateSchedulerEvent nodeResourceUpdatedEvent = (NodeResourceUpdateSchedulerEvent )@event; UpdateNodeResource(nodeResourceUpdatedEvent.GetRMNode(), nodeResourceUpdatedEvent .GetResourceOption()); break; } case SchedulerEventType.NodeUpdate: { NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)@event; NodeUpdate(nodeUpdatedEvent.GetRMNode()); break; } case SchedulerEventType.AppAdded: { AppAddedSchedulerEvent appAddedEvent = (AppAddedSchedulerEvent)@event; AddApplication(appAddedEvent.GetApplicationId(), appAddedEvent.GetQueue(), appAddedEvent .GetUser(), appAddedEvent.GetIsAppRecovering()); break; } case SchedulerEventType.AppRemoved: { AppRemovedSchedulerEvent appRemovedEvent = (AppRemovedSchedulerEvent)@event; DoneApplication(appRemovedEvent.GetApplicationID(), appRemovedEvent.GetFinalState ()); break; } case SchedulerEventType.AppAttemptAdded: { AppAttemptAddedSchedulerEvent appAttemptAddedEvent = (AppAttemptAddedSchedulerEvent )@event; AddApplicationAttempt(appAttemptAddedEvent.GetApplicationAttemptId(), appAttemptAddedEvent .GetTransferStateFromPreviousAttempt(), appAttemptAddedEvent.GetIsAttemptRecovering ()); break; } case SchedulerEventType.AppAttemptRemoved: { AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = (AppAttemptRemovedSchedulerEvent )@event; try { DoneApplicationAttempt(appAttemptRemovedEvent.GetApplicationAttemptID(), appAttemptRemovedEvent .GetFinalAttemptState(), appAttemptRemovedEvent.GetKeepContainersAcrossAppAttempts ()); } catch (IOException ie) { Log.Error("Unable to remove application " + appAttemptRemovedEvent.GetApplicationAttemptID (), ie); } break; } case SchedulerEventType.ContainerExpired: { ContainerExpiredSchedulerEvent containerExpiredEvent = (ContainerExpiredSchedulerEvent )@event; ContainerId containerid = containerExpiredEvent.GetContainerId(); CompletedContainer(GetRMContainer(containerid), SchedulerUtils.CreateAbnormalContainerStatus (containerid, SchedulerUtils.ExpiredContainer), RMContainerEventType.Expire); break; } case SchedulerEventType.ContainerRescheduled: { ContainerRescheduledEvent containerRescheduledEvent = (ContainerRescheduledEvent) @event; RMContainer container = containerRescheduledEvent.GetContainer(); RecoverResourceRequestForContainer(container); break; } default: { Log.Error("Invalid eventtype " + @event.GetType() + ". Ignoring!"); break; } } }
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException /// "/> /// <exception cref="System.Exception"/> /// <exception cref="Org.Apache.Hadoop.Security.AccessControlException"/> protected internal virtual void TestPlanFollower(bool isMove) { // Initialize plan based on move flag plan = new InMemoryPlan(scheduler.GetRootQueueMetrics(), policy, mAgent, scheduler .GetClusterResource(), 1L, res, scheduler.GetMinimumResourceCapability(), maxAlloc , "dedicated", null, isMove); // add a few reservations to the plan long ts = Runtime.CurrentTimeMillis(); ReservationId r1 = ReservationId.NewInstance(ts, 1); int[] f1 = new int[] { 10, 10, 10, 10, 10 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r1, null, "u3", "dedicated", 0, 0 + f1.Length, ReservationSystemTestUtil.GenerateAllocation (0L, 1L, f1), res, minAlloc))); ReservationId r2 = ReservationId.NewInstance(ts, 2); NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r2, null, "u3", "dedicated", 3, 3 + f1.Length, ReservationSystemTestUtil.GenerateAllocation (3L, 1L, f1), res, minAlloc))); ReservationId r3 = ReservationId.NewInstance(ts, 3); int[] f2 = new int[] { 0, 10, 20, 10, 0 }; NUnit.Framework.Assert.IsTrue(plan.ToString(), plan.AddReservation(new InMemoryReservationAllocation (r3, null, "u4", "dedicated", 10, 10 + f2.Length, ReservationSystemTestUtil.GenerateAllocation (10L, 1L, f2), res, minAlloc))); AbstractSchedulerPlanFollower planFollower = CreatePlanFollower(); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(0L); planFollower.Run(); Queue q = GetReservationQueue(r1.ToString()); AssertReservationQueueExists(r1); // submit an app to r1 string user_0 = "test-user"; ApplicationId appId = ApplicationId.NewInstance(0, 1); ApplicationAttemptId appAttemptId_0 = ApplicationAttemptId.NewInstance(appId, 0); AppAddedSchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, q.GetQueueName (), user_0); scheduler.Handle(addAppEvent); AppAttemptAddedSchedulerEvent appAttemptAddedEvent = new AppAttemptAddedSchedulerEvent (appAttemptId_0, false); scheduler.Handle(appAttemptAddedEvent); // initial default reservation queue should have no apps Queue defQ = GetDefaultQueue(); NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); AssertReservationQueueExists(r1, 0.1, 0.1); NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(q)); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueDoesNotExist(r3); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(3L); planFollower.Run(); NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); AssertReservationQueueExists(r1, 0.1, 0.1); NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(q)); AssertReservationQueueExists(r2, 0.1, 0.1); AssertReservationQueueDoesNotExist(r3); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(10L); planFollower.Run(); q = GetReservationQueue(r1.ToString()); if (isMove) { // app should have been moved to default reservation queue NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(defQ)); NUnit.Framework.Assert.IsNull(q); } else { // app should be killed NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); NUnit.Framework.Assert.IsNotNull(q); AppAttemptRemovedSchedulerEvent appAttemptRemovedEvent = new AppAttemptRemovedSchedulerEvent (appAttemptId_0, RMAppAttemptState.Killed, false); scheduler.Handle(appAttemptRemovedEvent); } AssertReservationQueueDoesNotExist(r2); AssertReservationQueueExists(r3, 0, 1.0); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(11L); planFollower.Run(); if (isMove) { // app should have been moved to default reservation queue NUnit.Framework.Assert.AreEqual(1, GetNumberOfApplications(defQ)); } else { // app should be killed NUnit.Framework.Assert.AreEqual(0, GetNumberOfApplications(defQ)); } AssertReservationQueueDoesNotExist(r1); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueExists(r3, 0.1, 0.1); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(12L); planFollower.Run(); AssertReservationQueueDoesNotExist(r1); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueExists(r3, 0.2, 0.2); Org.Mockito.Mockito.When(mClock.GetTime()).ThenReturn(16L); planFollower.Run(); AssertReservationQueueDoesNotExist(r1); AssertReservationQueueDoesNotExist(r2); AssertReservationQueueDoesNotExist(r3); VerifyCapacity(defQ); }
/// <exception cref="System.Exception"/> public virtual void TestHeadroom() { Configuration conf = new Configuration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); rm.Start(); FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler(); // Add a node RMNode n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, "127.0.0.2" ); fs.Handle(new NodeAddedSchedulerEvent(n1)); // Add two applications ApplicationId appId1 = BuilderUtils.NewApplicationId(100, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1, 1); CreateMockRMApp(appAttemptId1, rm.GetRMContext()); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user"); fs.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false ); fs.Handle(attemptEvent); ApplicationId appId2 = BuilderUtils.NewApplicationId(200, 2); ApplicationAttemptId appAttemptId2 = BuilderUtils.NewApplicationAttemptId(appId2, 1); CreateMockRMApp(appAttemptId2, rm.GetRMContext()); SchedulerEvent appEvent2 = new AppAddedSchedulerEvent(appId2, "queue", "user"); fs.Handle(appEvent2); SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false ); fs.Handle(attemptEvent2); IList <ContainerId> emptyId = new AList <ContainerId>(); IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>(); // Set up resource requests // Ask for a 1 GB container for app 1 IList <ResourceRequest> ask1 = new AList <ResourceRequest>(); ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest .Any, BuilderUtils.NewResource(Gb, 1), 1)); fs.Allocate(appAttemptId1, ask1, emptyId, null, null); // Ask for a 2 GB container for app 2 IList <ResourceRequest> ask2 = new AList <ResourceRequest>(); ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest .Any, BuilderUtils.NewResource(2 * Gb, 1), 1)); fs.Allocate(appAttemptId2, ask2, emptyId, null, null); // Trigger container assignment fs.Handle(new NodeUpdateSchedulerEvent(n1)); // Get the allocation for the applications and verify headroom Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation1.GetResourceLimit ().GetMemory()); Allocation allocation2 = fs.Allocate(appAttemptId2, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation2.GetResourceLimit ().GetMemory()); rm.Stop(); }
/// <exception cref="System.Exception"/> public virtual void TestBlackListNodes() { Configuration conf = new Configuration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); rm.Start(); FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler(); int rack_num_0 = 0; int rack_num_1 = 1; // Add 4 nodes in 2 racks // host_0_0 in rack0 string host_0_0 = "127.0.0.1"; RMNode n1 = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_0 ); fs.Handle(new NodeAddedSchedulerEvent(n1)); // host_0_1 in rack0 string host_0_1 = "127.0.0.2"; RMNode n2 = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_1 ); fs.Handle(new NodeAddedSchedulerEvent(n2)); // host_1_0 in rack1 string host_1_0 = "127.0.0.3"; RMNode n3 = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_0 ); fs.Handle(new NodeAddedSchedulerEvent(n3)); // host_1_1 in rack1 string host_1_1 = "127.0.0.4"; RMNode n4 = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_1 ); fs.Handle(new NodeAddedSchedulerEvent(n4)); // Add one application ApplicationId appId1 = BuilderUtils.NewApplicationId(100, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1, 1); CreateMockRMApp(appAttemptId1, rm.GetRMContext()); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user"); fs.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false ); fs.Handle(attemptEvent); IList <ContainerId> emptyId = new AList <ContainerId>(); IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>(); // Allow rack-locality for rack_1, but blacklist host_1_0 // Set up resource requests // Ask for a 1 GB container for app 1 IList <ResourceRequest> ask1 = new AList <ResourceRequest>(); ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), "rack1" , BuilderUtils.NewResource(Gb, 1), 1)); ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest .Any, BuilderUtils.NewResource(Gb, 1), 1)); fs.Allocate(appAttemptId1, ask1, emptyId, Sharpen.Collections.SingletonList(host_1_0 ), null); // Trigger container assignment fs.Handle(new NodeUpdateSchedulerEvent(n3)); // Get the allocation for the application and verify no allocation on blacklist node Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation1", 0, allocation1.GetContainers().Count ); // verify host_1_1 can get allocated as not in blacklist fs.Handle(new NodeUpdateSchedulerEvent(n4)); Allocation allocation2 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation2", 1, allocation2.GetContainers().Count ); IList <Container> containerList = allocation2.GetContainers(); foreach (Container container in containerList) { NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container.GetNodeId (), n4.GetNodeID()); } // Ask for a 1 GB container again for app 1 IList <ResourceRequest> ask2 = new AList <ResourceRequest>(); // this time, rack0 is also in blacklist, so only host_1_1 is available to // be assigned ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest .Any, BuilderUtils.NewResource(Gb, 1), 1)); fs.Allocate(appAttemptId1, ask2, emptyId, Sharpen.Collections.SingletonList("rack0" ), null); // verify n1 is not qualified to be allocated fs.Handle(new NodeUpdateSchedulerEvent(n1)); Allocation allocation3 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation3", 0, allocation3.GetContainers().Count ); // verify n2 is not qualified to be allocated fs.Handle(new NodeUpdateSchedulerEvent(n2)); Allocation allocation4 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation4", 0, allocation4.GetContainers().Count ); // verify n3 is not qualified to be allocated fs.Handle(new NodeUpdateSchedulerEvent(n3)); Allocation allocation5 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation5", 0, allocation5.GetContainers().Count ); fs.Handle(new NodeUpdateSchedulerEvent(n4)); Allocation allocation6 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation6", 1, allocation6.GetContainers().Count ); containerList = allocation6.GetContainers(); foreach (Container container_1 in containerList) { NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container_1.GetNodeId (), n4.GetNodeID()); } rm.Stop(); }
/// <exception cref="System.Exception"/> public virtual void TestUpdateResourceOnNode() { AsyncDispatcher dispatcher = new InlineDispatcher(); Configuration conf = new Configuration(); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager (conf); containerTokenSecretManager.RollMasterKey(); NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf ); nmTokenSecretManager.RollMasterKey(); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); FifoScheduler scheduler = new _FifoScheduler_275(this); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler); rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher >()); rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >()); ((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration()); scheduler.SetRMContext(rmContext); scheduler.Init(conf); scheduler.Start(); scheduler.Reinitialize(new Configuration(), rmContext); RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(2048, 4), 1, "127.0.0.1" ); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0); scheduler.Handle(nodeEvent1); MethodInfo method = Sharpen.Runtime.GetDeclaredMethod(scheduler.GetType(), "getNodes" ); IDictionary <NodeId, FiCaSchedulerNode> schedulerNodes = (IDictionary <NodeId, FiCaSchedulerNode >)method.Invoke(scheduler); NUnit.Framework.Assert.AreEqual(schedulerNodes.Values.Count, 1); Org.Apache.Hadoop.Yarn.Api.Records.Resource newResource = Resources.CreateResource (1024, 4); NodeResourceUpdateSchedulerEvent node0ResourceUpdate = new NodeResourceUpdateSchedulerEvent (node0, ResourceOption.NewInstance(newResource, RMNode.OverCommitTimeoutMillisDefault )); scheduler.Handle(node0ResourceUpdate); // SchedulerNode's total resource and available resource are changed. NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetTotalResource ().GetMemory(), 1024); NUnit.Framework.Assert.AreEqual(schedulerNodes[node0.GetNodeID()].GetAvailableResource ().GetMemory(), 1024); QueueInfo queueInfo = scheduler.GetQueueInfo(null, false, false); NUnit.Framework.Assert.AreEqual(0.0f, queueInfo.GetCurrentCapacity(), 0.0f); int _appId = 1; int _appAttemptId = 1; ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId); CreateMockRMApp(appAttemptId, rmContext); AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId (), "queue1", "user1"); scheduler.Handle(appEvent); AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId , false); scheduler.Handle(attemptEvent); int memory = 1024; int priority = 1; IList <ResourceRequest> ask = new AList <ResourceRequest>(); ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority , 1); ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority , 1); ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority , 1); ask.AddItem(nodeLocal); ask.AddItem(rackLocal); ask.AddItem(any); scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null); // Before the node update event, there are one local request NUnit.Framework.Assert.AreEqual(1, nodeLocal.GetNumContainers()); NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0); // Now schedule. scheduler.Handle(node0Update); // After the node update event, check no local request NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers()); // Also check that one container was scheduled SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId); NUnit.Framework.Assert.AreEqual(1, info.GetLiveContainers().Count); // And check the default Queue now is full. queueInfo = scheduler.GetQueueInfo(null, false, false); NUnit.Framework.Assert.AreEqual(1.0f, queueInfo.GetCurrentCapacity(), 0.0f); }
/// <exception cref="System.Exception"/> public virtual void TestNodeLocalAssignment() { AsyncDispatcher dispatcher = new InlineDispatcher(); Configuration conf = new Configuration(); RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager (conf); containerTokenSecretManager.RollMasterKey(); NMTokenSecretManagerInRM nmTokenSecretManager = new NMTokenSecretManagerInRM(conf ); nmTokenSecretManager.RollMasterKey(); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); FifoScheduler scheduler = new FifoScheduler(); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler); rmContext.SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher >()); rmContext.SetRMApplicationHistoryWriter(Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >()); ((RMContextImpl)rmContext).SetYarnConfiguration(new YarnConfiguration()); scheduler.SetRMContext(rmContext); scheduler.Init(conf); scheduler.Start(); scheduler.Reinitialize(new Configuration(), rmContext); RMNode node0 = MockNodes.NewNodeInfo(1, Resources.CreateResource(1024 * 64), 1, "127.0.0.1" ); NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node0); scheduler.Handle(nodeEvent1); int _appId = 1; int _appAttemptId = 1; ApplicationAttemptId appAttemptId = CreateAppAttemptId(_appId, _appAttemptId); CreateMockRMApp(appAttemptId, rmContext); AppAddedSchedulerEvent appEvent = new AppAddedSchedulerEvent(appAttemptId.GetApplicationId (), "queue1", "user1"); scheduler.Handle(appEvent); AppAttemptAddedSchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId , false); scheduler.Handle(attemptEvent); int memory = 64; int nConts = 3; int priority = 20; IList <ResourceRequest> ask = new AList <ResourceRequest>(); ResourceRequest nodeLocal = CreateResourceRequest(memory, node0.GetHostName(), priority , nConts); ResourceRequest rackLocal = CreateResourceRequest(memory, node0.GetRackName(), priority , nConts); ResourceRequest any = CreateResourceRequest(memory, ResourceRequest.Any, priority , nConts); ask.AddItem(nodeLocal); ask.AddItem(rackLocal); ask.AddItem(any); scheduler.Allocate(appAttemptId, ask, new AList <ContainerId>(), null, null); NodeUpdateSchedulerEvent node0Update = new NodeUpdateSchedulerEvent(node0); // Before the node update event, there are 3 local requests outstanding NUnit.Framework.Assert.AreEqual(3, nodeLocal.GetNumContainers()); scheduler.Handle(node0Update); // After the node update event, check that there are no more local requests // outstanding NUnit.Framework.Assert.AreEqual(0, nodeLocal.GetNumContainers()); //Also check that the containers were scheduled SchedulerAppReport info = scheduler.GetSchedulerAppInfo(appAttemptId); NUnit.Framework.Assert.AreEqual(3, info.GetLiveContainers().Count); scheduler.Stop(); }