public virtual void TestOffSwitchSchedulingMultiLevelQueues()
{
// Setup queue configs
SetupMultiLevelQueues(csConf);
//B3
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
.Root, queues, queues, TestUtils.spyHook);
// Setup some nodes
int memoryPerNode = 10;
int coresPerNode = 10;
int numNodes = 2;
FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode
* Gb);
FiCaSchedulerNode node_1 = TestUtils.GetMockNode("host_1", DefaultRack, 0, memoryPerNode
* Gb);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(numNodes * (memoryPerNode * Gb), numNodes * coresPerNode);
Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes);
// Start testing
LeafQueue b3 = (LeafQueue)queues[B3];
LeafQueue b2 = (LeafQueue)queues[B2];
// Simulate B3 returning a container on node_0
StubQueueAllocation(b2, clusterResource, node_0, 0 * Gb, NodeType.OffSwitch);
StubQueueAllocation(b3, clusterResource, node_0, 1 * Gb, NodeType.OffSwitch);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(b2, 0 * Gb, clusterResource);
VerifyQueueMetrics(b3, 1 * Gb, clusterResource);
// Now, B2 should get the scheduling opportunity since B2=0G/2G, B3=1G/7G
// also, B3 gets a scheduling opportunity since B2 allocates RACK_LOCAL
StubQueueAllocation(b2, clusterResource, node_1, 1 * Gb, NodeType.RackLocal);
StubQueueAllocation(b3, clusterResource, node_1, 1 * Gb, NodeType.OffSwitch);
root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource
));
InOrder allocationOrder = Org.Mockito.Mockito.InOrder(b2, b3);
allocationOrder.Verify(b2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(b3).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(b2, 1 * Gb, clusterResource);
VerifyQueueMetrics(b3, 2 * Gb, clusterResource);
// Now, B3 should get the scheduling opportunity
// since B2 has 1/2G while B3 has 2/7G,
// However, since B3 returns off-switch, B2 won't get an opportunity
StubQueueAllocation(b2, clusterResource, node_0, 1 * Gb, NodeType.NodeLocal);
StubQueueAllocation(b3, clusterResource, node_0, 1 * Gb, NodeType.OffSwitch);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
allocationOrder = Org.Mockito.Mockito.InOrder(b3, b2);
allocationOrder.Verify(b3).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(b2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(b2, 1 * Gb, clusterResource);
VerifyQueueMetrics(b3, 3 * Gb, clusterResource);
}
// check min depth is handled when -depth is specified
/// <exception cref="System.IO.IOException"/>
public virtual void ProcessArgumentsDepthFirstMinDepth()
{
List <PathData> items = CreateDirectories();
Org.Apache.Hadoop.FS.Shell.Find.Find find = new Org.Apache.Hadoop.FS.Shell.Find.Find
();
find.GetOptions().SetDepthFirst(true);
find.GetOptions().SetMinDepth(1);
find.SetConf(conf);
TextWriter @out = Org.Mockito.Mockito.Mock <TextWriter>();
find.GetOptions().SetOut(@out);
TextWriter err = Org.Mockito.Mockito.Mock <TextWriter>();
find.GetOptions().SetErr(err);
Expression expr = Org.Mockito.Mockito.Mock <Expression>();
Org.Mockito.Mockito.When(expr.Apply((PathData)Matchers.Any(), Matchers.AnyInt()))
.ThenReturn(Result.Pass);
TestFind.FileStatusChecker fsCheck = Org.Mockito.Mockito.Mock <TestFind.FileStatusChecker
>();
Expression test = new TestFind.TestExpression(this, expr, fsCheck);
find.SetRootExpression(test);
find.ProcessArguments(items);
InOrder inOrder = Org.Mockito.Mockito.InOrder(expr);
inOrder.Verify(expr).SetOptions(find.GetOptions());
inOrder.Verify(expr).Prepare();
inOrder.Verify(expr).Apply(item1aa, 2);
inOrder.Verify(expr).Apply(item1a, 1);
inOrder.Verify(expr).Apply(item1b, 1);
inOrder.Verify(expr).Apply(item5a, 1);
inOrder.Verify(expr).Apply(item5b, 1);
inOrder.Verify(expr).Apply(item5ca, 2);
inOrder.Verify(expr).Apply(item5c, 1);
inOrder.Verify(expr).Apply(item5d, 1);
inOrder.Verify(expr).Apply(item5e, 1);
inOrder.Verify(expr).Finish();
Org.Mockito.Mockito.VerifyNoMoreInteractions(expr);
InOrder inOrderFsCheck = Org.Mockito.Mockito.InOrder(fsCheck);
inOrderFsCheck.Verify(fsCheck).Check(item1aa.stat);
inOrderFsCheck.Verify(fsCheck).Check(item1a.stat);
inOrderFsCheck.Verify(fsCheck).Check(item1b.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5a.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5b.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5ca.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5c.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5d.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5e.stat);
Org.Mockito.Mockito.VerifyNoMoreInteractions(fsCheck);
Org.Mockito.Mockito.VerifyNoMoreInteractions(@out);
Org.Mockito.Mockito.VerifyNoMoreInteractions(err);
}
public virtual void TestConsecutiveFetch()
{
int MaxEventsToFetch = 100;
TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 1);
TaskUmbilicalProtocol umbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol>
();
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.AnyInt(), Matchers.AnyInt(), Matchers.Any <TaskAttemptID>())).ThenReturn
(GetMockedCompletionEventsUpdate(0, 0));
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.Eq(0), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid))).ThenReturn(GetMockedCompletionEventsUpdate
(0, MaxEventsToFetch));
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.Eq(MaxEventsToFetch), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid)))
.ThenReturn(GetMockedCompletionEventsUpdate(MaxEventsToFetch, MaxEventsToFetch));
Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(),
Matchers.Eq(MaxEventsToFetch * 2), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid
))).ThenReturn(GetMockedCompletionEventsUpdate(MaxEventsToFetch * 2, 3));
ShuffleScheduler <string, string> scheduler = Org.Mockito.Mockito.Mock <ShuffleScheduler
>();
ExceptionReporter reporter = Org.Mockito.Mockito.Mock <ExceptionReporter>();
TestEventFetcher.EventFetcherForTest <string, string> ef = new TestEventFetcher.EventFetcherForTest
<string, string>(tid, umbilical, scheduler, reporter, MaxEventsToFetch);
ef.GetMapCompletionEvents();
Org.Mockito.Mockito.Verify(reporter, Org.Mockito.Mockito.Never()).ReportException
(Matchers.Any <Exception>());
InOrder inOrder = Org.Mockito.Mockito.InOrder(umbilical);
inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq
(0), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid));
inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq
(MaxEventsToFetch), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid));
inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq
(MaxEventsToFetch * 2), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid));
Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(MaxEventsToFetch
* 2 + 3)).Resolve(Matchers.Any <TaskCompletionEvent>());
}
public virtual void TestWrite()
{
long length = 11111;
long fileCount = 22222;
long directoryCount = 33333;
long quota = 44444;
long spaceConsumed = 55555;
long spaceQuota = 66666;
ContentSummary contentSummary = new ContentSummary.Builder().Length(length).FileCount
(fileCount).DirectoryCount(directoryCount).Quota(quota).SpaceConsumed(spaceConsumed
).SpaceQuota(spaceQuota).Build();
BinaryWriter writer = Org.Mockito.Mockito.Mock <BinaryWriter>();
InOrder inOrder = Org.Mockito.Mockito.InOrder(@out);
contentSummary.Write(@out);
inOrder.Verify(@out).WriteLong(length);
inOrder.Verify(@out).WriteLong(fileCount);
inOrder.Verify(@out).WriteLong(directoryCount);
inOrder.Verify(@out).WriteLong(quota);
inOrder.Verify(@out).WriteLong(spaceConsumed);
inOrder.Verify(@out).WriteLong(spaceQuota);
}
public virtual void TestMultiLevelQueues()
{
/*
* Structure of queue:
* Root
* ____________
* / | \ \
* A B C D
* / | / | \ \
* A1 A2 B1 B2 B3 C1
* \
* C11
* \
* C111
* \
* C1111
*/
// Setup queue configs
SetupMultiLevelQueues(csConf);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
.Root, queues, queues, TestUtils.spyHook);
// Setup some nodes
int memoryPerNode = 10;
int coresPerNode = 16;
int numNodes = 3;
FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode
* Gb);
FiCaSchedulerNode node_1 = TestUtils.GetMockNode("host_1", DefaultRack, 0, memoryPerNode
* Gb);
FiCaSchedulerNode node_2 = TestUtils.GetMockNode("host_2", DefaultRack, 0, memoryPerNode
* Gb);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(numNodes * (memoryPerNode * Gb), numNodes * coresPerNode);
Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes);
// Start testing
CSQueue a = queues[A];
CSQueue b = queues[B];
CSQueue c = queues[C];
CSQueue d = queues[D];
CSQueue a1 = queues[A1];
CSQueue a2 = queues[A2];
CSQueue b1 = queues[B1];
CSQueue b2 = queues[B2];
CSQueue b3 = queues[B3];
// Simulate C returning a container on node_0
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(a, 0 * Gb, clusterResource);
VerifyQueueMetrics(b, 0 * Gb, clusterResource);
VerifyQueueMetrics(c, 1 * Gb, clusterResource);
VerifyQueueMetrics(d, 0 * Gb, clusterResource);
Org.Mockito.Mockito.Reset(a);
Org.Mockito.Mockito.Reset(b);
Org.Mockito.Mockito.Reset(c);
// Now get B2 to allocate
// A = 0/3, B = 0/15, C = 1/6, D=0/6
StubQueueAllocation(a, clusterResource, node_1, 0 * Gb);
StubQueueAllocation(b2, clusterResource, node_1, 4 * Gb);
StubQueueAllocation(c, clusterResource, node_1, 0 * Gb);
root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(a, 0 * Gb, clusterResource);
VerifyQueueMetrics(b, 4 * Gb, clusterResource);
VerifyQueueMetrics(c, 1 * Gb, clusterResource);
Org.Mockito.Mockito.Reset(a);
Org.Mockito.Mockito.Reset(b);
Org.Mockito.Mockito.Reset(c);
// Now get both A1, C & B3 to allocate in right order
// A = 0/3, B = 4/15, C = 1/6, D=0/6
StubQueueAllocation(a1, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b3, clusterResource, node_0, 2 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 2 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
InOrder allocationOrder = Org.Mockito.Mockito.InOrder(a, c, b);
allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(c).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(a, 1 * Gb, clusterResource);
VerifyQueueMetrics(b, 6 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
Org.Mockito.Mockito.Reset(a);
Org.Mockito.Mockito.Reset(b);
Org.Mockito.Mockito.Reset(c);
// Now verify max-capacity
// A = 1/3, B = 6/15, C = 3/6, D=0/6
// Ensure a1 won't alloc above max-cap although it should get
// scheduling opportunity now, right after a2
Log.Info("here");
((ParentQueue)a).SetMaxCapacity(.1f);
// a should be capped at 3/30
StubQueueAllocation(a1, clusterResource, node_2, 1 * Gb);
// shouldn't be
// allocated due
// to max-cap
StubQueueAllocation(a2, clusterResource, node_2, 2 * Gb);
StubQueueAllocation(b3, clusterResource, node_2, 1 * Gb);
StubQueueAllocation(b1, clusterResource, node_2, 1 * Gb);
StubQueueAllocation(c, clusterResource, node_2, 1 * Gb);
root.AssignContainers(clusterResource, node_2, new ResourceLimits(clusterResource
));
allocationOrder = Org.Mockito.Mockito.InOrder(a, a2, a1, b, c);
allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(a2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(c).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 8 * Gb, clusterResource);
VerifyQueueMetrics(c, 4 * Gb, clusterResource);
Org.Mockito.Mockito.Reset(a);
Org.Mockito.Mockito.Reset(b);
Org.Mockito.Mockito.Reset(c);
}
public virtual void TestSingleLevelQueues()
{
// Setup queue configs
SetupSingleLevelQueues(csConf);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
.Root, queues, queues, TestUtils.spyHook);
// Setup some nodes
int memoryPerNode = 10;
int coresPerNode = 16;
int numNodes = 2;
FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode
* Gb);
FiCaSchedulerNode node_1 = TestUtils.GetMockNode("host_1", DefaultRack, 0, memoryPerNode
* Gb);
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(numNodes * (memoryPerNode * Gb), numNodes * coresPerNode);
Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes);
// Start testing
LeafQueue a = (LeafQueue)queues[A];
LeafQueue b = (LeafQueue)queues[B];
// Simulate B returning a container on node_0
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(a, 0 * Gb, clusterResource);
VerifyQueueMetrics(b, 1 * Gb, clusterResource);
// Now, A should get the scheduling opportunity since A=0G/6G, B=1G/14G
StubQueueAllocation(a, clusterResource, node_1, 2 * Gb);
StubQueueAllocation(b, clusterResource, node_1, 1 * Gb);
root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource
));
InOrder allocationOrder = Org.Mockito.Mockito.InOrder(a, b);
allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(a, 2 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
// Now, B should get the scheduling opportunity
// since A has 2/6G while B has 2/14G
StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 2 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
allocationOrder = Org.Mockito.Mockito.InOrder(b, a);
allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 4 * Gb, clusterResource);
// Now, B should still get the scheduling opportunity
// since A has 3/6G while B has 4/14G
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 4 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
allocationOrder = Org.Mockito.Mockito.InOrder(b, a);
allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 8 * Gb, clusterResource);
// Now, A should get the scheduling opportunity
// since A has 3/6G while B has 8/14G
StubQueueAllocation(a, clusterResource, node_1, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_1, 1 * Gb);
root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource
));
allocationOrder = Org.Mockito.Mockito.InOrder(a, b);
allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), AnyResourceLimits());
VerifyQueueMetrics(a, 4 * Gb, clusterResource);
VerifyQueueMetrics(b, 9 * Gb, clusterResource);
}
public virtual void TestSortedQueues()
{
// Setup queue configs
SetupSortedQueues(csConf);
IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
.Root, queues, queues, TestUtils.spyHook);
// Setup some nodes
int memoryPerNode = 10;
int coresPerNode = 16;
int numNodes = 1;
FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode
* Gb);
Org.Mockito.Mockito.DoNothing().When(node_0).ReleaseContainer(Matchers.Any <Container
>());
Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
(numNodes * (memoryPerNode * Gb), numNodes * coresPerNode);
Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes);
// Start testing
CSQueue a = queues[A];
CSQueue b = queues[B];
CSQueue c = queues[C];
CSQueue d = queues[D];
string user_0 = "user_0";
// Stub an App and its containerCompleted
FiCaSchedulerApp app_0 = GetMockApplication(0, user_0);
Org.Mockito.Mockito.DoReturn(true).When(app_0).ContainerCompleted(Matchers.Any <RMContainer
>(), Matchers.Any <ContainerStatus>(), Matchers.Any <RMContainerEventType>());
Priority priority = TestUtils.CreateMockPriority(1);
ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
>();
DrainDispatcher drainDispatcher = new DrainDispatcher();
RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
>();
SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
>();
RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();
Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
);
Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
);
Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
);
Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
());
ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app_0.GetApplicationId
(), 1);
ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
Container container = TestUtils.GetMockContainer(containerId, node_0.GetNodeID(),
Resources.CreateResource(1 * Gb), priority);
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, node_0.GetNodeID
(), "user", rmContext);
// Assign {1,2,3,4} 1GB containers respectively to queues
StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
for (int i = 0; i < 2; i++)
{
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
for (int i_1 = 0; i_1 < 3; i_1++)
{
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
for (int i_2 = 0; i_2 < 4; i_2++)
{
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 1 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
VerifyQueueMetrics(a, 1 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 4 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
//Release 3 x 1GB containers from D
for (int i_3 = 0; i_3 < 3; i_3++)
{
d.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
.Kill, null, true);
}
VerifyQueueMetrics(a, 1 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
//reset manually resources on node
node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 1 - 2 -
3 - 1) * Gb);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Assign 2 x 1GB Containers to A
for (int i_4 = 0; i_4 < 2; i_4++)
{
StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
}
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
//Release 1GB Container from A
a.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
.Kill, null, true);
VerifyQueueMetrics(a, 2 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
//reset manually resources on node
node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 -
3 - 1) * Gb);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Assign 1GB container to B
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(a, 2 * Gb, clusterResource);
VerifyQueueMetrics(b, 3 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
//Release 1GB container resources from B
b.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
.Kill, null, true);
VerifyQueueMetrics(a, 2 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
//reset manually resources on node
node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 -
3 - 1) * Gb);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Assign 1GB container to A
StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 1 * Gb, clusterResource);
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
// Now do the real test, where B and D request a 1GB container
// D should should get the next container if the order is correct
StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
StubQueueAllocation(d, clusterResource, node_0, 1 * Gb);
root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
));
InOrder allocationOrder = Org.Mockito.Mockito.InOrder(d, b);
allocationOrder.Verify(d).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>());
allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
<FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>());
VerifyQueueMetrics(a, 3 * Gb, clusterResource);
VerifyQueueMetrics(b, 2 * Gb, clusterResource);
VerifyQueueMetrics(c, 3 * Gb, clusterResource);
VerifyQueueMetrics(d, 2 * Gb, clusterResource);
//D got the container
Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
}
// check symlinks given as path arguments are processed correctly with the
// follow option
/// <exception cref="System.IO.IOException"/>
public virtual void ProcessArgumentsOptionFollow()
{
List <PathData> items = CreateDirectories();
Org.Apache.Hadoop.FS.Shell.Find.Find find = new Org.Apache.Hadoop.FS.Shell.Find.Find
();
find.GetOptions().SetFollowLink(true);
find.SetConf(conf);
TextWriter @out = Org.Mockito.Mockito.Mock <TextWriter>();
find.GetOptions().SetOut(@out);
TextWriter err = Org.Mockito.Mockito.Mock <TextWriter>();
find.GetOptions().SetErr(err);
Expression expr = Org.Mockito.Mockito.Mock <Expression>();
Org.Mockito.Mockito.When(expr.Apply((PathData)Matchers.Any(), Matchers.AnyInt()))
.ThenReturn(Result.Pass);
TestFind.FileStatusChecker fsCheck = Org.Mockito.Mockito.Mock <TestFind.FileStatusChecker
>();
Expression test = new TestFind.TestExpression(this, expr, fsCheck);
find.SetRootExpression(test);
find.ProcessArguments(items);
InOrder inOrder = Org.Mockito.Mockito.InOrder(expr);
inOrder.Verify(expr).SetOptions(find.GetOptions());
inOrder.Verify(expr).Prepare();
inOrder.Verify(expr).Apply(item1, 0);
inOrder.Verify(expr).Apply(item1a, 1);
inOrder.Verify(expr).Apply(item1aa, 2);
inOrder.Verify(expr).Apply(item1b, 1);
inOrder.Verify(expr).Apply(item2, 0);
inOrder.Verify(expr).Apply(item3, 0);
inOrder.Verify(expr).Apply(item4, 0);
inOrder.Verify(expr).Apply(item5, 0);
inOrder.Verify(expr).Apply(item5a, 1);
inOrder.Verify(expr).Apply(item5b, 1);
// triggers infinite loop message
inOrder.Verify(expr).Apply(item5c, 1);
inOrder.Verify(expr).Apply(item5ca, 2);
inOrder.Verify(expr).Apply(item5d, 1);
inOrder.Verify(expr).Apply(item5ca, 2);
// following item5d symlink
inOrder.Verify(expr).Apply(item5e, 1);
inOrder.Verify(expr).Finish();
Org.Mockito.Mockito.VerifyNoMoreInteractions(expr);
InOrder inOrderFsCheck = Org.Mockito.Mockito.InOrder(fsCheck);
inOrderFsCheck.Verify(fsCheck).Check(item1.stat);
inOrderFsCheck.Verify(fsCheck).Check(item1a.stat);
inOrderFsCheck.Verify(fsCheck).Check(item1aa.stat);
inOrderFsCheck.Verify(fsCheck).Check(item1b.stat);
inOrderFsCheck.Verify(fsCheck).Check(item2.stat);
inOrderFsCheck.Verify(fsCheck, Org.Mockito.Mockito.Times(2)).Check(item3.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5.stat);
inOrderFsCheck.Verify(fsCheck).Check(item1b.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5c.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5ca.stat);
inOrderFsCheck.Verify(fsCheck).Check(item5c.stat);
inOrderFsCheck.Verify(fsCheck, Org.Mockito.Mockito.Times(2)).Check(item5ca.stat);
Org.Mockito.Mockito.VerifyNoMoreInteractions(fsCheck);
Org.Mockito.Mockito.VerifyNoMoreInteractions(@out);
Org.Mockito.Mockito.Verify(err).WriteLine("Infinite loop ignored: " + item5b.ToString
() + " -> " + item5.ToString());
Org.Mockito.Mockito.VerifyNoMoreInteractions(err);
}
