public virtual void TestUpdateMaxAllocationUsesTotal() { int configuredMaxVCores = 20; int configuredMaxMemory = 10 * 1024; Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(configuredMaxMemory, configuredMaxVCores); ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores ); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); MockRM rm = new MockRM(conf); try { rm.Start(); AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler( ); Org.Apache.Hadoop.Yarn.Api.Records.Resource emptyResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(0, 0); Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1024, 5); Org.Apache.Hadoop.Yarn.Api.Records.Resource fullResource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2048, 10); SchedulerNode mockNode1 = Org.Mockito.Mockito.Mock <SchedulerNode>(); Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("foo" , 8080)); Org.Mockito.Mockito.When(mockNode1.GetAvailableResource()).ThenReturn(emptyResource ); Org.Mockito.Mockito.When(mockNode1.GetTotalResource()).ThenReturn(fullResource1); SchedulerNode mockNode2 = Org.Mockito.Mockito.Mock <SchedulerNode>(); Org.Mockito.Mockito.When(mockNode1.GetNodeID()).ThenReturn(NodeId.NewInstance("bar" , 8081)); Org.Mockito.Mockito.When(mockNode2.GetAvailableResource()).ThenReturn(emptyResource ); Org.Mockito.Mockito.When(mockNode2.GetTotalResource()).ThenReturn(fullResource2); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); scheduler.nodes = new Dictionary <NodeId, SchedulerNode>(); scheduler.nodes[mockNode1.GetNodeID()] = mockNode1; scheduler.UpdateMaximumAllocation(mockNode1, true); VerifyMaximumResourceCapability(fullResource1, scheduler); scheduler.nodes[mockNode2.GetNodeID()] = mockNode2; scheduler.UpdateMaximumAllocation(mockNode2, true); VerifyMaximumResourceCapability(fullResource2, scheduler); Sharpen.Collections.Remove(scheduler.nodes, mockNode2.GetNodeID()); scheduler.UpdateMaximumAllocation(mockNode2, false); VerifyMaximumResourceCapability(fullResource1, scheduler); Sharpen.Collections.Remove(scheduler.nodes, mockNode1.GetNodeID()); scheduler.UpdateMaximumAllocation(mockNode1, false); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); } finally { rm.Stop(); } }
private void VerifyMaximumResourceCapability(Org.Apache.Hadoop.Yarn.Api.Records.Resource expectedMaximumResource, AbstractYarnScheduler scheduler) { Org.Apache.Hadoop.Yarn.Api.Records.Resource schedulerMaximumResourceCapability = scheduler.GetMaximumResourceCapability(); NUnit.Framework.Assert.AreEqual(expectedMaximumResource.GetMemory(), schedulerMaximumResourceCapability .GetMemory()); NUnit.Framework.Assert.AreEqual(expectedMaximumResource.GetVirtualCores(), schedulerMaximumResourceCapability .GetVirtualCores()); }
public virtual void TestMaxAllocationAfterUpdateNodeResource() { int configuredMaxVCores = 20; int configuredMaxMemory = 10 * 1024; Org.Apache.Hadoop.Yarn.Api.Records.Resource configuredMaximumResource = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(configuredMaxMemory, configuredMaxVCores); ConfigureScheduler(); YarnConfiguration conf = GetConf(); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationVcores, configuredMaxVCores ); conf.SetInt(YarnConfiguration.RmSchedulerMaximumAllocationMb, configuredMaxMemory ); conf.SetLong(YarnConfiguration.RmWorkPreservingRecoverySchedulingWaitMs, 0); MockRM rm = new MockRM(conf); try { rm.Start(); AbstractYarnScheduler scheduler = (AbstractYarnScheduler)rm.GetResourceScheduler( ); VerifyMaximumResourceCapability(configuredMaximumResource, scheduler); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource1 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(2048, 5); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource2 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(4096, 10); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource3 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(512, 1); Org.Apache.Hadoop.Yarn.Api.Records.Resource resource4 = Org.Apache.Hadoop.Yarn.Api.Records.Resource .NewInstance(1024, 2); RMNode node1 = MockNodes.NewNodeInfo(0, resource1, 1, "127.0.0.2"); scheduler.Handle(new NodeAddedSchedulerEvent(node1)); RMNode node2 = MockNodes.NewNodeInfo(0, resource3, 2, "127.0.0.3"); scheduler.Handle(new NodeAddedSchedulerEvent(node2)); VerifyMaximumResourceCapability(resource1, scheduler); // increase node1 resource scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource2, 0)); VerifyMaximumResourceCapability(resource2, scheduler); // decrease node1 resource scheduler.UpdateNodeResource(node1, ResourceOption.NewInstance(resource1, 0)); VerifyMaximumResourceCapability(resource1, scheduler); // increase node2 resource scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource4, 0)); VerifyMaximumResourceCapability(resource1, scheduler); // decrease node2 resource scheduler.UpdateNodeResource(node2, ResourceOption.NewInstance(resource3, 0)); VerifyMaximumResourceCapability(resource1, scheduler); } finally { rm.Stop(); } }
/// <exception cref="System.Exception"/> private void TestMaximumAllocationMemoryHelper(AbstractYarnScheduler scheduler, int node1MaxMemory, int node2MaxMemory, int node3MaxMemory, params int[] expectedMaxMemory ) { NUnit.Framework.Assert.AreEqual(6, expectedMaxMemory.Length); NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes()); int maxMemory = scheduler.GetMaximumResourceCapability().GetMemory(); NUnit.Framework.Assert.AreEqual(expectedMaxMemory[0], maxMemory); RMNode node1 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node1MaxMemory), 1, "127.0.0.2"); scheduler.Handle(new NodeAddedSchedulerEvent(node1)); NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes()); maxMemory = scheduler.GetMaximumResourceCapability().GetMemory(); NUnit.Framework.Assert.AreEqual(expectedMaxMemory[1], maxMemory); scheduler.Handle(new NodeRemovedSchedulerEvent(node1)); NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes()); maxMemory = scheduler.GetMaximumResourceCapability().GetMemory(); NUnit.Framework.Assert.AreEqual(expectedMaxMemory[2], maxMemory); RMNode node2 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node2MaxMemory), 2, "127.0.0.3"); scheduler.Handle(new NodeAddedSchedulerEvent(node2)); NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes()); maxMemory = scheduler.GetMaximumResourceCapability().GetMemory(); NUnit.Framework.Assert.AreEqual(expectedMaxMemory[3], maxMemory); RMNode node3 = MockNodes.NewNodeInfo(0, Resources.CreateResource(node3MaxMemory), 3, "127.0.0.4"); scheduler.Handle(new NodeAddedSchedulerEvent(node3)); NUnit.Framework.Assert.AreEqual(2, scheduler.GetNumClusterNodes()); maxMemory = scheduler.GetMaximumResourceCapability().GetMemory(); NUnit.Framework.Assert.AreEqual(expectedMaxMemory[4], maxMemory); scheduler.Handle(new NodeRemovedSchedulerEvent(node3)); NUnit.Framework.Assert.AreEqual(1, scheduler.GetNumClusterNodes()); maxMemory = scheduler.GetMaximumResourceCapability().GetMemory(); NUnit.Framework.Assert.AreEqual(expectedMaxMemory[5], maxMemory); scheduler.Handle(new NodeRemovedSchedulerEvent(node2)); NUnit.Framework.Assert.AreEqual(0, scheduler.GetNumClusterNodes()); }
public _TimerTask_450(AbstractYarnScheduler <T, N> _enclosing) { this._enclosing = _enclosing; }