public virtual void TestInitialize()
{
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
CapacityScheduler capScheduler = null;
try
{
capScheduler = testUtil.MockCapacityScheduler(10);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
CapacityReservationSystem reservationSystem = new CapacityReservationSystem();
reservationSystem.SetRMContext(capScheduler.GetRMContext());
try
{
reservationSystem.Reinitialize(capScheduler.GetConf(), capScheduler.GetRMContext(
));
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
string planQName = testUtil.GetreservationQueueName();
ReservationSystemTestUtil.ValidateReservationQueue(reservationSystem, planQName);
}
public virtual void TestQueueParsing()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue b1 = capacityScheduler.GetQueue("b1");
NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
(), Delta);
CSQueue c12 = capacityScheduler.GetQueue("c12");
NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
);
NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
), Delta);
ServiceOperations.StopQuietly(capacityScheduler);
}
/// <summary>
/// Test init a queue configuration, children's capacity for a given label
/// doesn't equals to 100%.
/// </summary>
/// <remarks>
/// Test init a queue configuration, children's capacity for a given label
/// doesn't equals to 100%. This expect IllegalArgumentException thrown.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual void TestQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard
()
{
nodeLabelManager.AddToCluserNodeLabels(ImmutableSet.Of("red", "blue"));
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
SetupQueueConfigurationWithLabels(csConf);
csConf.SetCapacityByLabel(CapacitySchedulerConfiguration.Root + ".b.b3", "red", 24
);
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, ImmutableSet.
Of(RMNodeLabelsManager.Any));
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root + ".b", ImmutableSet
.Of(RMNodeLabelsManager.Any));
CapacityScheduler capacityScheduler = new CapacityScheduler();
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetConf(csConf);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(csConf);
capacityScheduler.Start();
ServiceOperations.StopQuietly(capacityScheduler);
}
public virtual void TestQueueCapacityZero()
{
// Setup queue configs
SetupMultiLevelQueues(csConf);
// set parent and child capacity to 0
string QB = CapacitySchedulerConfiguration.Root + "." + B;
csConf.SetCapacity(QB, 0);
csConf.SetCapacity(QB + "." + B1, 0);
csConf.SetCapacity(QB + "." + B2, 0);
csConf.SetCapacity(QB + "." + B3, 0);
string QA = CapacitySchedulerConfiguration.Root + "." + A;
csConf.SetCapacity(QA, 60);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
try
{
CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
.Root, queues, queues, TestUtils.spyHook);
}
catch (ArgumentException e)
{
NUnit.Framework.Assert.Fail("Failed to create queues with 0 capacity: " + e);
}
NUnit.Framework.Assert.IsTrue("Failed to create queues with 0 capacity", true);
}
public virtual void TestOffSwitchSchedulingMultiLevelQueues()
{
// Setup queue configs
SetupMultiLevelQueues(csConf);
//B3
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
.Root, queues, queues, TestUtils.spyHook);
// Setup some nodes
int memoryPerNode = 10;
int coresPerNode = 10;
int numNodes = 2;
FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode
* Gb);
FiCaSchedulerNode node_1 = TestUtils.GetMockNode("host_1", DefaultRack, 0, memoryPerNode
* Gb);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(numNodes * (memoryPerNode * Gb), numNodes * coresPerNode);
Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes);
// Start testing
LeafQueue b3 = (LeafQueue)queues[B3];
LeafQueue b2 = (LeafQueue)queues[B2];
// Simulate B3 returning a container on node_0
StubQueueAllocation(b2, clusterResource, node_0, 0 * Gb, NodeType.OffSwitch);
StubQueueAllocation(b3, clusterResource, node_0, 1 * Gb, NodeType.OffSwitch);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(b2, 0 * Gb, clusterResource);
VerifyQueueMetrics(b3, 1 * Gb, clusterResource);
// Now, B2 should get the scheduling opportunity since B2=0G/2G, B3=1G/7G
// also, B3 gets a scheduling opportunity since B2 allocates RACK_LOCAL
StubQueueAllocation(b2, clusterResource, node_1, 1 * Gb, NodeType.RackLocal);
StubQueueAllocation(b3, clusterResource, node_1, 1 * Gb, NodeType.OffSwitch);
root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource
));
InOrder allocationOrder = Org.Mockito.Mockito.InOrder(b2, b3);
allocationOrder.Verify(b2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(b3).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(b2, 1 * Gb, clusterResource);
VerifyQueueMetrics(b3, 2 * Gb, clusterResource);
// Now, B3 should get the scheduling opportunity
// since B2 has 1/2G while B3 has 2/7G,
// However, since B3 returns off-switch, B2 won't get an opportunity
StubQueueAllocation(b2, clusterResource, node_0, 1 * Gb, NodeType.NodeLocal);
StubQueueAllocation(b3, clusterResource, node_0, 1 * Gb, NodeType.OffSwitch);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
allocationOrder = Org.Mockito.Mockito.InOrder(b3, b2);
allocationOrder.Verify(b3).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(b2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(b2, 1 * Gb, clusterResource);
VerifyQueueMetrics(b3, 3 * Gb, clusterResource);
}
public virtual void TestNestedQueueParsingShouldTrimSpaces()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupNestedQueueConfigurationWithSpacesShouldBeTrimmed(csConf);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
capacityScheduler.SetRMContext(TestUtils.GetMockRMContext());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, TestUtils.GetMockRMContext());
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.IsNotNull(a);
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue c = capacityScheduler.GetQueue("c");
NUnit.Framework.Assert.IsNotNull(c);
NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.70, c.GetAbsoluteMaximumCapacity(), Delta);
CSQueue a1 = capacityScheduler.GetQueue("a1");
NUnit.Framework.Assert.IsNotNull(a1);
NUnit.Framework.Assert.AreEqual(0.10 * 0.6, a1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a1.GetAbsoluteMaximumCapacity(), Delta);
CSQueue a2 = capacityScheduler.GetQueue("a2");
NUnit.Framework.Assert.IsNotNull(a2);
NUnit.Framework.Assert.AreEqual(0.10 * 0.4, a2.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a2.GetAbsoluteMaximumCapacity(), Delta);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestQueueParsingWithMoveQueue()
{
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(conf);
csConf.SetQueues("root", new string[] { "a" });
csConf.SetQueues("root.a", new string[] { "x", "y" });
csConf.SetCapacity("root.a", 100);
csConf.SetCapacity("root.a.x", 50);
csConf.SetCapacity("root.a.y", 50);
CapacityScheduler capacityScheduler = new CapacityScheduler();
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetConf(csConf);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(csConf);
capacityScheduler.Start();
csConf.SetQueues("root", new string[] { "a", "x" });
csConf.SetQueues("root.a", new string[] { "y" });
csConf.SetCapacity("root.x", 50);
csConf.SetCapacity("root.a", 50);
csConf.SetCapacity("root.a.y", 100);
capacityScheduler.Reinitialize(csConf, rmContext);
}
public virtual void TestAdminRefreshQueuesWithFileSystemBasedConfigurationProvider
()
{
configuration.Set(YarnConfiguration.RmConfigurationProviderClass, "org.apache.hadoop.yarn.FileSystemBasedConfigurationProvider"
);
//upload default configurations
UploadDefaultConfiguration();
try
{
rm = new MockRM(configuration);
rm.Init(configuration);
rm.Start();
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Should not get any exceptions");
}
CapacityScheduler cs = (CapacityScheduler)rm.GetRMContext().GetScheduler();
int maxAppsBefore = cs.GetConfiguration().GetMaximumSystemApplications();
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
csConf.Set("yarn.scheduler.capacity.maximum-applications", "5000");
UploadConfiguration(csConf, "capacity-scheduler.xml");
rm.adminService.RefreshQueues(RefreshQueuesRequest.NewInstance());
int maxAppsAfter = cs.GetConfiguration().GetMaximumSystemApplications();
NUnit.Framework.Assert.AreEqual(maxAppsAfter, 5000);
NUnit.Framework.Assert.IsTrue(maxAppsAfter != maxAppsBefore);
}
/// <exception cref="System.IO.IOException"/>
private void CheckQMapping(string user, string expected, CapacityScheduler cs)
{
string actual = cs.GetMappedQueueForTest(user);
NUnit.Framework.Assert.IsTrue("expected " + expected + " actual " + actual, expected
.Equals(actual));
}
// Test RM restarts after AM container is preempted, new RM should not count
// AM preemption failure towards the max-retry-account and should be able to
// re-launch the AM.
/// <exception cref="System.Exception"/>
public virtual void TestPreemptedAMRestartOnRMRestart()
{
YarnConfiguration conf = new YarnConfiguration();
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(ResourceScheduler
));
conf.SetBoolean(YarnConfiguration.RecoveryEnabled, true);
conf.SetBoolean(YarnConfiguration.RmWorkPreservingRecoveryEnabled, false);
conf.Set(YarnConfiguration.RmStore, typeof(MemoryRMStateStore).FullName);
// explicitly set max-am-retry count as 1.
conf.SetInt(YarnConfiguration.RmAmMaxAttempts, 1);
MemoryRMStateStore memStore = new MemoryRMStateStore();
memStore.Init(conf);
MockRM rm1 = new MockRM(conf, memStore);
rm1.Start();
MockNM nm1 = new MockNM("127.0.0.1:1234", 8000, rm1.GetResourceTrackerService());
nm1.RegisterNode();
RMApp app1 = rm1.SubmitApp(200);
RMAppAttempt attempt1 = app1.GetCurrentAppAttempt();
MockAM am1 = MockRM.LaunchAndRegisterAM(app1, rm1, nm1);
CapacityScheduler scheduler = (CapacityScheduler)rm1.GetResourceScheduler();
ContainerId amContainer = ContainerId.NewContainerId(am1.GetApplicationAttemptId(
), 1);
// Forcibly preempt the am container;
scheduler.KillContainer(scheduler.GetRMContainer(amContainer));
am1.WaitForState(RMAppAttemptState.Failed);
NUnit.Framework.Assert.IsTrue(!attempt1.ShouldCountTowardsMaxAttemptRetry());
rm1.WaitForState(app1.GetApplicationId(), RMAppState.Accepted);
// state store has 1 attempt stored.
ApplicationStateData appState = memStore.GetState().GetApplicationState()[app1.GetApplicationId
()];
NUnit.Framework.Assert.AreEqual(1, appState.GetAttemptCount());
// attempt stored has the preempted container exit status.
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Preempted, appState.GetAttempt
(am1.GetApplicationAttemptId()).GetAMContainerExitStatus());
// Restart rm.
MockRM rm2 = new MockRM(conf, memStore);
nm1.SetResourceTrackerService(rm2.GetResourceTrackerService());
nm1.RegisterNode();
rm2.Start();
// Restarted RM should re-launch the am.
MockAM am2 = rm2.WaitForNewAMToLaunchAndRegister(app1.GetApplicationId(), 2, nm1);
MockRM.FinishAMAndVerifyAppState(app1, rm2, nm1, am2);
RMAppAttempt attempt2 = rm2.GetRMContext().GetRMApps()[app1.GetApplicationId()].GetCurrentAppAttempt
();
NUnit.Framework.Assert.IsTrue(attempt2.ShouldCountTowardsMaxAttemptRetry());
NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Invalid, appState.GetAttempt(
am2.GetApplicationAttemptId()).GetAMContainerExitStatus());
rm1.Stop();
rm2.Stop();
}
private void CheckUsedResource(MockRM rm, string queueName, int memory, string label
)
{
CapacityScheduler scheduler = (CapacityScheduler)rm.GetResourceScheduler();
CSQueue queue = scheduler.GetQueue(queueName);
NUnit.Framework.Assert.AreEqual(memory, queue.GetQueueResourceUsage().GetUsed(label
).GetMemory());
}
public virtual void TestMoveAppToPlanQueue()
{
CapacityScheduler scheduler = (CapacityScheduler)rm.GetResourceScheduler();
// submit an app
RMApp app = rm.SubmitApp(Gb, "test-move-1", "user_0", null, "b1");
ApplicationAttemptId appAttemptId = rm.GetApplicationReport(app.GetApplicationId(
)).GetCurrentApplicationAttemptId();
// check preconditions
IList <ApplicationAttemptId> appsInB1 = scheduler.GetAppsInQueue("b1");
NUnit.Framework.Assert.AreEqual(1, appsInB1.Count);
IList <ApplicationAttemptId> appsInB = scheduler.GetAppsInQueue("b");
NUnit.Framework.Assert.AreEqual(1, appsInB.Count);
NUnit.Framework.Assert.IsTrue(appsInB.Contains(appAttemptId));
IList <ApplicationAttemptId> appsInA = scheduler.GetAppsInQueue("a");
NUnit.Framework.Assert.IsTrue(appsInA.IsEmpty());
string queue = scheduler.GetApplicationAttempt(appsInB1[0]).GetQueue().GetQueueName
();
NUnit.Framework.Assert.IsTrue(queue.Equals("b1"));
IList <ApplicationAttemptId> appsInRoot = scheduler.GetAppsInQueue("root");
NUnit.Framework.Assert.IsTrue(appsInRoot.Contains(appAttemptId));
NUnit.Framework.Assert.AreEqual(1, appsInRoot.Count);
// create the default reservation queue
string defQName = "a" + ReservationConstants.DefaultQueueSuffix;
ReservationQueue defQ = new ReservationQueue(scheduler, defQName, (PlanQueue)scheduler
.GetQueue("a"));
scheduler.AddQueue(defQ);
defQ.SetEntitlement(new QueueEntitlement(1f, 1f));
IList <ApplicationAttemptId> appsInDefQ = scheduler.GetAppsInQueue(defQName);
NUnit.Framework.Assert.IsTrue(appsInDefQ.IsEmpty());
// now move the app to plan queue
scheduler.MoveApplication(app.GetApplicationId(), "a");
// check postconditions
appsInDefQ = scheduler.GetAppsInQueue(defQName);
NUnit.Framework.Assert.AreEqual(1, appsInDefQ.Count);
queue = scheduler.GetApplicationAttempt(appsInDefQ[0]).GetQueue().GetQueueName();
NUnit.Framework.Assert.IsTrue(queue.Equals(defQName));
appsInA = scheduler.GetAppsInQueue("a");
NUnit.Framework.Assert.IsTrue(appsInA.Contains(appAttemptId));
NUnit.Framework.Assert.AreEqual(1, appsInA.Count);
appsInRoot = scheduler.GetAppsInQueue("root");
NUnit.Framework.Assert.IsTrue(appsInRoot.Contains(appAttemptId));
NUnit.Framework.Assert.AreEqual(1, appsInRoot.Count);
appsInB1 = scheduler.GetAppsInQueue("b1");
NUnit.Framework.Assert.IsTrue(appsInB1.IsEmpty());
appsInB = scheduler.GetAppsInQueue("b");
NUnit.Framework.Assert.IsTrue(appsInB.IsEmpty());
rm.Stop();
}
/// <exception cref="System.Exception"/>
public virtual void TestMaxCapacity()
{
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
conf.SetQueues(CapacitySchedulerConfiguration.Root, new string[] { "a", "b", "c" }
);
string A = CapacitySchedulerConfiguration.Root + ".a";
conf.SetCapacity(A, 50);
conf.SetMaximumCapacity(A, 60);
string B = CapacitySchedulerConfiguration.Root + ".b";
conf.SetCapacity(B, 50);
conf.SetMaximumCapacity(B, 45);
// Should throw an exception
bool fail = false;
CapacityScheduler capacityScheduler;
try
{
capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(new YarnConfiguration());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, null);
}
catch (ArgumentException)
{
fail = true;
}
NUnit.Framework.Assert.IsTrue("Didn't throw IllegalArgumentException for wrong maxCap"
, fail);
conf.SetMaximumCapacity(B, 60);
// Now this should work
capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(new YarnConfiguration());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, null);
fail = false;
try
{
LeafQueue a = (LeafQueue)capacityScheduler.GetQueue(A);
a.SetMaxCapacity(45);
}
catch (ArgumentException)
{
fail = true;
}
NUnit.Framework.Assert.IsTrue("Didn't throw IllegalArgumentException for wrong "
+ "setMaxCap", fail);
capacityScheduler.Stop();
}
public virtual void TestReinitialize()
{
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
CapacityScheduler capScheduler = null;
try
{
capScheduler = testUtil.MockCapacityScheduler(10);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
CapacityReservationSystem reservationSystem = new CapacityReservationSystem();
CapacitySchedulerConfiguration conf = capScheduler.GetConfiguration();
RMContext mockContext = capScheduler.GetRMContext();
reservationSystem.SetRMContext(mockContext);
try
{
reservationSystem.Reinitialize(capScheduler.GetConfiguration(), mockContext);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
// Assert queue in original config
string planQName = testUtil.GetreservationQueueName();
ReservationSystemTestUtil.ValidateReservationQueue(reservationSystem, planQName);
// Dynamically add a plan
string newQ = "reservation";
NUnit.Framework.Assert.IsNull(reservationSystem.GetPlan(newQ));
testUtil.UpdateQueueConfiguration(conf, newQ);
try
{
capScheduler.Reinitialize(conf, mockContext);
}
catch (IOException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
try
{
reservationSystem.Reinitialize(conf, mockContext);
}
catch (YarnException e)
{
NUnit.Framework.Assert.Fail(e.Message);
}
ReservationSystemTestUtil.ValidateNewReservationQueue(reservationSystem, newQ);
}
// Test even if AM container is allocated with containerId not equal to 1, the
// following allocate requests from AM should be able to retrieve the
// corresponding NM Token.
/// <exception cref="System.Exception"/>
public virtual void TestNMTokenSentForNormalContainer()
{
conf.Set(YarnConfiguration.RmScheduler, typeof(CapacityScheduler).GetCanonicalName
());
MockRM rm = new MockRM(conf);
rm.Start();
MockNM nm1 = rm.RegisterNode("h1:1234", 5120);
RMApp app = rm.SubmitApp(2000);
RMAppAttempt attempt = app.GetCurrentAppAttempt();
// Call getNewContainerId to increase container Id so that the AM container
// Id doesn't equal to one.
CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
cs.GetApplicationAttempt(attempt.GetAppAttemptId()).GetNewContainerId();
// kick the scheduling
nm1.NodeHeartbeat(true);
MockAM am = MockRM.LaunchAM(app, rm, nm1);
// am container Id not equal to 1.
NUnit.Framework.Assert.IsTrue(attempt.GetMasterContainer().GetId().GetContainerId
() != 1);
// NMSecretManager doesn't record the node on which the am is allocated.
NUnit.Framework.Assert.IsFalse(rm.GetRMContext().GetNMTokenSecretManager().IsApplicationAttemptNMTokenPresent
(attempt.GetAppAttemptId(), nm1.GetNodeId()));
am.RegisterAppAttempt();
rm.WaitForState(app.GetApplicationId(), RMAppState.Running);
int NumContainers = 1;
IList <Container> containers = new AList <Container>();
// nmTokens keeps track of all the nmTokens issued in the allocate call.
IList <NMToken> expectedNMTokens = new AList <NMToken>();
// am1 allocate 1 container on nm1.
while (true)
{
AllocateResponse response = am.Allocate("127.0.0.1", 2000, NumContainers, new AList
<ContainerId>());
nm1.NodeHeartbeat(true);
Sharpen.Collections.AddAll(containers, response.GetAllocatedContainers());
Sharpen.Collections.AddAll(expectedNMTokens, response.GetNMTokens());
if (containers.Count == NumContainers)
{
break;
}
Sharpen.Thread.Sleep(200);
System.Console.Out.WriteLine("Waiting for container to be allocated.");
}
NodeId nodeId = expectedNMTokens[0].GetNodeId();
// NMToken is sent for the allocated container.
NUnit.Framework.Assert.AreEqual(nm1.GetNodeId(), nodeId);
}
public override void Init(Clock clock, ResourceScheduler sched, ICollection <Plan>
plans)
{
base.Init(clock, sched, plans);
Log.Info("Initializing Plan Follower Policy:" + this.GetType().GetCanonicalName()
);
if (!(sched is CapacityScheduler))
{
throw new YarnRuntimeException("CapacitySchedulerPlanFollower can only work with CapacityScheduler"
);
}
this.cs = (CapacityScheduler)sched;
}
public virtual void TestCompareRMNodeAfterReconnect()
{
Configuration yarnConf = new YarnConfiguration();
CapacityScheduler scheduler = new CapacityScheduler();
scheduler.SetConf(yarnConf);
ConfigurationProvider configurationProvider = ConfigurationProviderFactory.GetConfigurationProvider
(yarnConf);
configurationProvider.Init(yarnConf);
context.SetConfigurationProvider(configurationProvider);
RMNodeLabelsManager nlm = new RMNodeLabelsManager();
nlm.Init(yarnConf);
nlm.Start();
context.SetNodeLabelManager(nlm);
scheduler.SetRMContext(context);
scheduler.Init(yarnConf);
scheduler.Start();
dispatcher.Register(typeof(SchedulerEventType), scheduler);
string hostname1 = "localhost1";
Resource capability = BuilderUtils.NewResource(4096, 4);
RegisterNodeManagerRequest request1 = recordFactory.NewRecordInstance <RegisterNodeManagerRequest
>();
NodeId nodeId1 = NodeId.NewInstance(hostname1, 0);
request1.SetNodeId(nodeId1);
request1.SetHttpPort(0);
request1.SetResource(capability);
resourceTrackerService.RegisterNodeManager(request1);
NUnit.Framework.Assert.IsNotNull(context.GetRMNodes()[nodeId1]);
// verify Scheduler and RMContext use same RMNode reference.
NUnit.Framework.Assert.IsTrue(scheduler.GetSchedulerNode(nodeId1).GetRMNode() ==
context.GetRMNodes()[nodeId1]);
NUnit.Framework.Assert.AreEqual(context.GetRMNodes()[nodeId1].GetTotalCapability(
), capability);
Resource capability1 = BuilderUtils.NewResource(2048, 2);
request1.SetResource(capability1);
resourceTrackerService.RegisterNodeManager(request1);
NUnit.Framework.Assert.IsNotNull(context.GetRMNodes()[nodeId1]);
// verify Scheduler and RMContext use same RMNode reference
// after reconnect.
NUnit.Framework.Assert.IsTrue(scheduler.GetSchedulerNode(nodeId1).GetRMNode() ==
context.GetRMNodes()[nodeId1]);
// verify RMNode's capability is changed.
NUnit.Framework.Assert.AreEqual(context.GetRMNodes()[nodeId1].GetTotalCapability(
), capability1);
nlm.Stop();
scheduler.Stop();
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
public override void Reinitialize(Configuration conf, RMContext rmContext)
{
// Validate if the scheduler is capacity based
ResourceScheduler scheduler = rmContext.GetScheduler();
if (!(scheduler is CapacityScheduler))
{
throw new YarnRuntimeException("Class " + scheduler.GetType().GetCanonicalName()
+ " not instance of " + typeof(CapacityScheduler).GetCanonicalName());
}
capScheduler = (CapacityScheduler)scheduler;
this.conf = conf;
base.Reinitialize(conf, rmContext);
}
/// <exception cref="System.Exception"/>
public virtual void TestRootQueueParsing()
{
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
// non-100 percent value will throw IllegalArgumentException
conf.SetCapacity(CapacitySchedulerConfiguration.Root, 90);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(new YarnConfiguration());
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, null);
ServiceOperations.StopQuietly(capacityScheduler);
}
/// <exception cref="System.Exception"/>
public virtual void TestQueueCapacitySettingChildZero()
{
// Setup queue configs
SetupMultiLevelQueues(csConf);
// set child queues capacity to 0 when parents not 0
string QB = CapacitySchedulerConfiguration.Root + "." + B;
csConf.SetCapacity(QB + "." + B1, 0);
csConf.SetCapacity(QB + "." + B2, 0);
csConf.SetCapacity(QB + "." + B3, 0);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
.Root, queues, queues, TestUtils.spyHook);
}
/// <exception cref="Org.Apache.Hadoop.Yarn.Server.Resourcemanager.Reservation.Exceptions.PlanningException
/// "/>
/// <exception cref="System.IO.IOException"/>
public virtual void TestStress(int numJobs)
{
long timeWindow = 1000000L;
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterCapacity = Org.Apache.Hadoop.Yarn.Api.Records.Resource
.NewInstance(500 * 100 * 1024, 500 * 32);
step = 1000L;
ReservationSystemTestUtil testUtil = new ReservationSystemTestUtil();
CapacityScheduler scheduler = testUtil.MockCapacityScheduler(500 * 100);
string reservationQ = testUtil.GetFullReservationQueueName();
float instConstraint = 100;
float avgConstraint = 100;
ReservationSchedulerConfiguration conf = ReservationSystemTestUtil.CreateConf(reservationQ
, timeWindow, instConstraint, avgConstraint);
CapacityOverTimePolicy policy = new CapacityOverTimePolicy();
policy.Init(reservationQ, conf);
plan = new InMemoryPlan(scheduler.GetRootQueueMetrics(), policy, agent, clusterCapacity
, step, res, minAlloc, maxAlloc, "dedicated", null, true);
int acc = 0;
IList <ReservationDefinition> list = new AList <ReservationDefinition>();
for (long i = 0; i < numJobs; i++)
{
list.AddItem(ReservationSystemTestUtil.GenerateRandomRR(rand, i));
}
long start = Runtime.CurrentTimeMillis();
for (int i_1 = 0; i_1 < numJobs; i_1++)
{
try
{
if (agent.CreateReservation(ReservationSystemTestUtil.GetNewReservationId(), "u"
+ i_1 % 100, plan, list[i_1]))
{
acc++;
}
}
catch (PlanningException)
{
}
}
// ignore exceptions
long end = Runtime.CurrentTimeMillis();
System.Console.Out.WriteLine("Submitted " + numJobs + " jobs " + " accepted " + acc
+ " in " + (end - start) + "ms");
}
public virtual void TestQueueParsingWithUnusedLabels()
{
ImmutableSet <string> labels = ImmutableSet.Of("red", "blue");
// Initialize a cluster with labels, but doesn't use them, reinitialize
// shouldn't fail
nodeLabelManager.AddToCluserNodeLabels(labels);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(csConf);
csConf.SetAccessibleNodeLabels(CapacitySchedulerConfiguration.Root, labels);
YarnConfiguration conf = new YarnConfiguration(csConf);
CapacityScheduler capacityScheduler = new CapacityScheduler();
capacityScheduler.SetConf(conf);
RMContextImpl rmContext = new RMContextImpl(null, null, null, null, null, null, new
RMContainerTokenSecretManager(csConf), new NMTokenSecretManagerInRM(csConf), new
ClientToAMTokenSecretManagerInRM(), null);
rmContext.SetNodeLabelManager(nodeLabelManager);
capacityScheduler.SetRMContext(rmContext);
capacityScheduler.Init(conf);
capacityScheduler.Start();
capacityScheduler.Reinitialize(conf, rmContext);
// check root queue's capacity by label -- they should be all zero
CSQueue root = capacityScheduler.GetQueue(CapacitySchedulerConfiguration.Root);
NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("red"),
Delta);
NUnit.Framework.Assert.AreEqual(0, root.GetQueueCapacities().GetCapacity("blue"),
Delta);
CSQueue a = capacityScheduler.GetQueue("a");
NUnit.Framework.Assert.AreEqual(0.10, a.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual(0.15, a.GetAbsoluteMaximumCapacity(), Delta);
CSQueue b1 = capacityScheduler.GetQueue("b1");
NUnit.Framework.Assert.AreEqual(0.2 * 0.5, b1.GetAbsoluteCapacity(), Delta);
NUnit.Framework.Assert.AreEqual("Parent B has no MAX_CAP", 0.85, b1.GetAbsoluteMaximumCapacity
(), Delta);
CSQueue c12 = capacityScheduler.GetQueue("c12");
NUnit.Framework.Assert.AreEqual(0.7 * 0.5 * 0.45, c12.GetAbsoluteCapacity(), Delta
);
NUnit.Framework.Assert.AreEqual(0.7 * 0.55 * 0.7, c12.GetAbsoluteMaximumCapacity(
), Delta);
capacityScheduler.Stop();
}
/// <exception cref="System.IO.IOException"/>
private void CheckInvalidQMapping(YarnConfiguration conf, CapacityScheduler cs, string
mapping, string reason)
{
bool fail = false;
try
{
conf.Set(CapacitySchedulerConfiguration.QueueMapping, mapping);
cs.Reinitialize(conf, null);
}
catch (IOException)
{
fail = true;
}
NUnit.Framework.Assert.IsTrue("invalid mapping did not throw exception for " + reason
, fail);
}
private void CheckQueueLabelsInheritConfig(CapacityScheduler capacityScheduler)
{
// queue-A is red, blue
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-A1 inherits A's configuration
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a1").GetAccessibleNodeLabels
().ContainsAll(ImmutableSet.Of("red", "blue")));
// queue-A2 is "red"
NUnit.Framework.Assert.AreEqual(1, capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels
().Count);
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("a2").GetAccessibleNodeLabels
().Contains("red"));
// queue-B is "red"/"blue"
NUnit.Framework.Assert.IsTrue(capacityScheduler.GetQueue("b").GetAccessibleNodeLabels
().IsEmpty());
}
public virtual void TestSchedulerEventDispatcherForPreemptionEvents()
{
AsyncDispatcher rmDispatcher = new AsyncDispatcher();
CapacityScheduler sched = Org.Mockito.Mockito.Spy(new CapacityScheduler());
YarnConfiguration conf = new YarnConfiguration();
ResourceManager.SchedulerEventDispatcher schedulerDispatcher = new ResourceManager.SchedulerEventDispatcher
(sched);
rmDispatcher.Register(typeof(SchedulerEventType), schedulerDispatcher);
rmDispatcher.Init(conf);
rmDispatcher.Start();
schedulerDispatcher.Init(conf);
schedulerDispatcher.Start();
try
{
ApplicationAttemptId appAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId
>();
RMContainer container = Org.Mockito.Mockito.Mock <RMContainer>();
ContainerPreemptEvent event1 = new ContainerPreemptEvent(appAttemptId, container,
SchedulerEventType.DropReservation);
rmDispatcher.GetEventHandler().Handle(event1);
ContainerPreemptEvent event2 = new ContainerPreemptEvent(appAttemptId, container,
SchedulerEventType.KillContainer);
rmDispatcher.GetEventHandler().Handle(event2);
ContainerPreemptEvent event3 = new ContainerPreemptEvent(appAttemptId, container,
SchedulerEventType.PreemptContainer);
rmDispatcher.GetEventHandler().Handle(event3);
// Wait for events to be processed by scheduler dispatcher.
Sharpen.Thread.Sleep(1000);
Org.Mockito.Mockito.Verify(sched, Org.Mockito.Mockito.Times(3)).Handle(Matchers.Any
<SchedulerEvent>());
Org.Mockito.Mockito.Verify(sched).DropContainerReservation(container);
Org.Mockito.Mockito.Verify(sched).PreemptContainer(appAttemptId, container);
Org.Mockito.Mockito.Verify(sched).KillContainer(container);
}
catch (Exception)
{
NUnit.Framework.Assert.Fail();
}
finally
{
schedulerDispatcher.Stop();
rmDispatcher.Stop();
}
}
public virtual void SetUp()
{
CapacityScheduler spyCs = new CapacityScheduler();
cs = Org.Mockito.Mockito.Spy(spyCs);
scheduler = cs;
rmContext = TestUtils.GetMockRMContext();
spyRMContext = Org.Mockito.Mockito.Spy(rmContext);
ConcurrentMap <ApplicationId, RMApp> spyApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
<ApplicationId, RMApp>());
RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();
Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
())).ThenReturn(null);
Org.Mockito.Mockito.DoReturn(rmApp).When(spyApps)[(ApplicationId)Matchers.Any()];
Org.Mockito.Mockito.When(spyRMContext.GetRMApps()).ThenReturn(spyApps);
Org.Mockito.Mockito.When(spyRMContext.GetScheduler()).ThenReturn(scheduler);
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
ReservationSystemTestUtil.SetupQueueConfiguration(csConf);
cs.SetConf(csConf);
csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(minAlloc
);
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(maxAlloc
);
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource
(100 * 16 * Gb, 100 * 32));
Org.Mockito.Mockito.When(scheduler.GetClusterResource()).ThenReturn(Resources.CreateResource
(125 * Gb, 125));
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(new DefaultResourceCalculator
());
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(csConf);
containerTokenSecretManager.RollMasterKey();
Org.Mockito.Mockito.When(csContext.GetContainerTokenSecretManager()).ThenReturn(containerTokenSecretManager
);
cs.SetRMContext(spyRMContext);
cs.Init(csConf);
cs.Start();
SetupPlanFollower();
}
/// <exception cref="System.IO.IOException"/>
public static CapacityScheduler MockCapacityScheduler()
{
// stolen from TestCapacityScheduler
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
SetupQueueConfiguration(conf);
CapacityScheduler cs = new CapacityScheduler();
cs.SetConf(new YarnConfiguration());
RMContext rmContext = new RMContextImpl(null, null, null, null, null, null, new RMContainerTokenSecretManager
(conf), new NMTokenSecretManagerInRM(conf), new ClientToAMTokenSecretManagerInRM
(), null);
rmContext.SetNodeLabelManager(new NullRMNodeLabelsManager());
cs.SetRMContext(rmContext);
cs.Init(conf);
return(cs);
}
public virtual void SetUp()
{
CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
YarnConfiguration conf = new YarnConfiguration();
SetupQueueConfiguration(csConf);
rmContext = TestUtils.GetMockRMContext();
CapacitySchedulerContext csContext = Org.Mockito.Mockito.Mock <CapacitySchedulerContext
>();
Org.Mockito.Mockito.When(csContext.GetConfiguration()).ThenReturn(csConf);
Org.Mockito.Mockito.When(csContext.GetConf()).ThenReturn(conf);
Org.Mockito.Mockito.When(csContext.GetMinimumResourceCapability()).ThenReturn(Resources
.CreateResource(Gb, 1));
Org.Mockito.Mockito.When(csContext.GetMaximumResourceCapability()).ThenReturn(Resources
.CreateResource(16 * Gb, 32));
Org.Mockito.Mockito.When(csContext.GetClusterResource()).ThenReturn(Resources.CreateResource
(10 * 16 * Gb, 10 * 32));
Org.Mockito.Mockito.When(csContext.GetApplicationComparator()).ThenReturn(CapacityScheduler
.applicationComparator);
Org.Mockito.Mockito.When(csContext.GetQueueComparator()).ThenReturn(CapacityScheduler
.queueComparator);
Org.Mockito.Mockito.When(csContext.GetResourceCalculator()).ThenReturn(resourceCalculator
);
Org.Mockito.Mockito.When(csContext.GetRMContext()).ThenReturn(rmContext);
RMContainerTokenSecretManager containerTokenSecretManager = new RMContainerTokenSecretManager
(conf);
containerTokenSecretManager.RollMasterKey();
Org.Mockito.Mockito.When(csContext.GetContainerTokenSecretManager()).ThenReturn(containerTokenSecretManager
);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, "root", queues
, queues, TestUtils.spyHook);
queue = Org.Mockito.Mockito.Spy(new LeafQueue(csContext, A, root, null));
// Stub out ACL checks
Org.Mockito.Mockito.DoReturn(true).When(queue).HasAccess(Matchers.Any <QueueACL>()
, Matchers.Any <UserGroupInformation>());
// Some default values
Org.Mockito.Mockito.DoReturn(100).When(queue).GetMaxApplications();
Org.Mockito.Mockito.DoReturn(25).When(queue).GetMaxApplicationsPerUser();
}
public virtual void TestRemoveQueue()
{
CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
// Test add one reservation dynamically and manually modify capacity
ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a"));
cs.AddQueue(a1);
a1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f));
// submit an app
RMApp app = rm.SubmitApp(Gb, "test-move-1", "user_0", null, "a1");
// check preconditions
IList <ApplicationAttemptId> appsInA1 = cs.GetAppsInQueue("a1");
NUnit.Framework.Assert.AreEqual(1, appsInA1.Count);
try
{
cs.RemoveQueue("a1");
NUnit.Framework.Assert.Fail();
}
catch (SchedulerDynamicEditException)
{
}
// expected a1 contains applications
// clear queue by killling all apps
cs.KillAllAppsInQueue("a1");
// wait for events of move to propagate
rm.WaitForState(app.GetApplicationId(), RMAppState.Killed);
try
{
cs.RemoveQueue("a1");
NUnit.Framework.Assert.Fail();
}
catch (SchedulerDynamicEditException)
{
}
// expected a1 is not zero capacity
// set capacity to zero
cs.SetEntitlement("a1", new QueueEntitlement(0f, 0f));
cs.RemoveQueue("a1");
NUnit.Framework.Assert.IsTrue(cs.GetQueue("a1") == null);
rm.Stop();
}
public virtual void TestAddQueueFailCases()
{
CapacityScheduler cs = (CapacityScheduler)rm.GetResourceScheduler();
try
{
// Test invalid addition (adding non-zero size queue)
ReservationQueue a1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a"));
a1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f));
cs.AddQueue(a1);
NUnit.Framework.Assert.Fail();
}
catch (Exception)
{
}
// expected
// Test add one reservation dynamically and manually modify capacity
ReservationQueue a1_1 = new ReservationQueue(cs, "a1", (PlanQueue)cs.GetQueue("a"
));
cs.AddQueue(a1_1);
a1_1.SetEntitlement(new QueueEntitlement(A1Capacity / 100, 1f));
// Test add another reservation queue and use setEntitlement to modify
// capacity
ReservationQueue a2 = new ReservationQueue(cs, "a2", (PlanQueue)cs.GetQueue("a"));
cs.AddQueue(a2);
try
{
// Test invalid entitlement (sum of queues exceed 100%)
cs.SetEntitlement("a2", new QueueEntitlement(A2Capacity / 100 + 0.1f, 1.0f));
NUnit.Framework.Assert.Fail();
}
catch (Exception)
{
}
// expected
cs.SetEntitlement("a2", new QueueEntitlement(A2Capacity / 100, 1.0f));
// Verify all allocations match
tcs.CheckQueueCapacities(cs, ACapacity, BCapacity);
cs.Stop();
}