public virtual void TestOffSwitchSchedulingMultiLevelQueues() { // Setup queue configs SetupMultiLevelQueues(csConf); //B3 IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>(); CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration .Root, queues, queues, TestUtils.spyHook); // Setup some nodes int memoryPerNode = 10; int coresPerNode = 10; int numNodes = 2; FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode * Gb); FiCaSchedulerNode node_1 = TestUtils.GetMockNode("host_1", DefaultRack, 0, memoryPerNode * Gb); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (numNodes * (memoryPerNode * Gb), numNodes * coresPerNode); Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes); // Start testing LeafQueue b3 = (LeafQueue)queues[B3]; LeafQueue b2 = (LeafQueue)queues[B2]; // Simulate B3 returning a container on node_0 StubQueueAllocation(b2, clusterResource, node_0, 0 * Gb, NodeType.OffSwitch); StubQueueAllocation(b3, clusterResource, node_0, 1 * Gb, NodeType.OffSwitch); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); VerifyQueueMetrics(b2, 0 * Gb, clusterResource); VerifyQueueMetrics(b3, 1 * Gb, clusterResource); // Now, B2 should get the scheduling opportunity since B2=0G/2G, B3=1G/7G // also, B3 gets a scheduling opportunity since B2 allocates RACK_LOCAL StubQueueAllocation(b2, clusterResource, node_1, 1 * Gb, NodeType.RackLocal); StubQueueAllocation(b3, clusterResource, node_1, 1 * Gb, NodeType.OffSwitch); root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource )); InOrder allocationOrder = Org.Mockito.Mockito.InOrder(b2, b3); allocationOrder.Verify(b2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(b3).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); VerifyQueueMetrics(b2, 1 * Gb, clusterResource); VerifyQueueMetrics(b3, 2 * Gb, clusterResource); // Now, B3 should get the scheduling opportunity // since B2 has 1/2G while B3 has 2/7G, // However, since B3 returns off-switch, B2 won't get an opportunity StubQueueAllocation(b2, clusterResource, node_0, 1 * Gb, NodeType.NodeLocal); StubQueueAllocation(b3, clusterResource, node_0, 1 * Gb, NodeType.OffSwitch); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); allocationOrder = Org.Mockito.Mockito.InOrder(b3, b2); allocationOrder.Verify(b3).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(b2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); VerifyQueueMetrics(b2, 1 * Gb, clusterResource); VerifyQueueMetrics(b3, 3 * Gb, clusterResource); }
// check min depth is handled when -depth is specified /// <exception cref="System.IO.IOException"/> public virtual void ProcessArgumentsDepthFirstMinDepth() { List <PathData> items = CreateDirectories(); Org.Apache.Hadoop.FS.Shell.Find.Find find = new Org.Apache.Hadoop.FS.Shell.Find.Find (); find.GetOptions().SetDepthFirst(true); find.GetOptions().SetMinDepth(1); find.SetConf(conf); TextWriter @out = Org.Mockito.Mockito.Mock <TextWriter>(); find.GetOptions().SetOut(@out); TextWriter err = Org.Mockito.Mockito.Mock <TextWriter>(); find.GetOptions().SetErr(err); Expression expr = Org.Mockito.Mockito.Mock <Expression>(); Org.Mockito.Mockito.When(expr.Apply((PathData)Matchers.Any(), Matchers.AnyInt())) .ThenReturn(Result.Pass); TestFind.FileStatusChecker fsCheck = Org.Mockito.Mockito.Mock <TestFind.FileStatusChecker >(); Expression test = new TestFind.TestExpression(this, expr, fsCheck); find.SetRootExpression(test); find.ProcessArguments(items); InOrder inOrder = Org.Mockito.Mockito.InOrder(expr); inOrder.Verify(expr).SetOptions(find.GetOptions()); inOrder.Verify(expr).Prepare(); inOrder.Verify(expr).Apply(item1aa, 2); inOrder.Verify(expr).Apply(item1a, 1); inOrder.Verify(expr).Apply(item1b, 1); inOrder.Verify(expr).Apply(item5a, 1); inOrder.Verify(expr).Apply(item5b, 1); inOrder.Verify(expr).Apply(item5ca, 2); inOrder.Verify(expr).Apply(item5c, 1); inOrder.Verify(expr).Apply(item5d, 1); inOrder.Verify(expr).Apply(item5e, 1); inOrder.Verify(expr).Finish(); Org.Mockito.Mockito.VerifyNoMoreInteractions(expr); InOrder inOrderFsCheck = Org.Mockito.Mockito.InOrder(fsCheck); inOrderFsCheck.Verify(fsCheck).Check(item1aa.stat); inOrderFsCheck.Verify(fsCheck).Check(item1a.stat); inOrderFsCheck.Verify(fsCheck).Check(item1b.stat); inOrderFsCheck.Verify(fsCheck).Check(item5a.stat); inOrderFsCheck.Verify(fsCheck).Check(item5b.stat); inOrderFsCheck.Verify(fsCheck).Check(item5ca.stat); inOrderFsCheck.Verify(fsCheck).Check(item5c.stat); inOrderFsCheck.Verify(fsCheck).Check(item5d.stat); inOrderFsCheck.Verify(fsCheck).Check(item5e.stat); Org.Mockito.Mockito.VerifyNoMoreInteractions(fsCheck); Org.Mockito.Mockito.VerifyNoMoreInteractions(@out); Org.Mockito.Mockito.VerifyNoMoreInteractions(err); }
public virtual void TestConsecutiveFetch() { int MaxEventsToFetch = 100; TaskAttemptID tid = new TaskAttemptID("12345", 1, TaskType.Reduce, 1, 1); TaskUmbilicalProtocol umbilical = Org.Mockito.Mockito.Mock <TaskUmbilicalProtocol> (); Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.AnyInt(), Matchers.AnyInt(), Matchers.Any <TaskAttemptID>())).ThenReturn (GetMockedCompletionEventsUpdate(0, 0)); Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq(0), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid))).ThenReturn(GetMockedCompletionEventsUpdate (0, MaxEventsToFetch)); Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid))) .ThenReturn(GetMockedCompletionEventsUpdate(MaxEventsToFetch, MaxEventsToFetch)); Org.Mockito.Mockito.When(umbilical.GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq(MaxEventsToFetch * 2), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid ))).ThenReturn(GetMockedCompletionEventsUpdate(MaxEventsToFetch * 2, 3)); ShuffleScheduler <string, string> scheduler = Org.Mockito.Mockito.Mock <ShuffleScheduler >(); ExceptionReporter reporter = Org.Mockito.Mockito.Mock <ExceptionReporter>(); TestEventFetcher.EventFetcherForTest <string, string> ef = new TestEventFetcher.EventFetcherForTest <string, string>(tid, umbilical, scheduler, reporter, MaxEventsToFetch); ef.GetMapCompletionEvents(); Org.Mockito.Mockito.Verify(reporter, Org.Mockito.Mockito.Never()).ReportException (Matchers.Any <Exception>()); InOrder inOrder = Org.Mockito.Mockito.InOrder(umbilical); inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq (0), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid)); inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq (MaxEventsToFetch), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid)); inOrder.Verify(umbilical).GetMapCompletionEvents(Matchers.Any <JobID>(), Matchers.Eq (MaxEventsToFetch * 2), Matchers.Eq(MaxEventsToFetch), Matchers.Eq(tid)); Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(MaxEventsToFetch * 2 + 3)).Resolve(Matchers.Any <TaskCompletionEvent>()); }
public virtual void TestWrite() { long length = 11111; long fileCount = 22222; long directoryCount = 33333; long quota = 44444; long spaceConsumed = 55555; long spaceQuota = 66666; ContentSummary contentSummary = new ContentSummary.Builder().Length(length).FileCount (fileCount).DirectoryCount(directoryCount).Quota(quota).SpaceConsumed(spaceConsumed ).SpaceQuota(spaceQuota).Build(); BinaryWriter writer = Org.Mockito.Mockito.Mock <BinaryWriter>(); InOrder inOrder = Org.Mockito.Mockito.InOrder(@out); contentSummary.Write(@out); inOrder.Verify(@out).WriteLong(length); inOrder.Verify(@out).WriteLong(fileCount); inOrder.Verify(@out).WriteLong(directoryCount); inOrder.Verify(@out).WriteLong(quota); inOrder.Verify(@out).WriteLong(spaceConsumed); inOrder.Verify(@out).WriteLong(spaceQuota); }
public virtual void TestMultiLevelQueues() { /* * Structure of queue: * Root * ____________ * / | \ \ * A B C D * / | / | \ \ * A1 A2 B1 B2 B3 C1 * \ * C11 * \ * C111 * \ * C1111 */ // Setup queue configs SetupMultiLevelQueues(csConf); IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>(); CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration .Root, queues, queues, TestUtils.spyHook); // Setup some nodes int memoryPerNode = 10; int coresPerNode = 16; int numNodes = 3; FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode * Gb); FiCaSchedulerNode node_1 = TestUtils.GetMockNode("host_1", DefaultRack, 0, memoryPerNode * Gb); FiCaSchedulerNode node_2 = TestUtils.GetMockNode("host_2", DefaultRack, 0, memoryPerNode * Gb); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (numNodes * (memoryPerNode * Gb), numNodes * coresPerNode); Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes); // Start testing CSQueue a = queues[A]; CSQueue b = queues[B]; CSQueue c = queues[C]; CSQueue d = queues[D]; CSQueue a1 = queues[A1]; CSQueue a2 = queues[A2]; CSQueue b1 = queues[B1]; CSQueue b2 = queues[B2]; CSQueue b3 = queues[B3]; // Simulate C returning a container on node_0 StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 1 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); VerifyQueueMetrics(a, 0 * Gb, clusterResource); VerifyQueueMetrics(b, 0 * Gb, clusterResource); VerifyQueueMetrics(c, 1 * Gb, clusterResource); VerifyQueueMetrics(d, 0 * Gb, clusterResource); Org.Mockito.Mockito.Reset(a); Org.Mockito.Mockito.Reset(b); Org.Mockito.Mockito.Reset(c); // Now get B2 to allocate // A = 0/3, B = 0/15, C = 1/6, D=0/6 StubQueueAllocation(a, clusterResource, node_1, 0 * Gb); StubQueueAllocation(b2, clusterResource, node_1, 4 * Gb); StubQueueAllocation(c, clusterResource, node_1, 0 * Gb); root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource )); VerifyQueueMetrics(a, 0 * Gb, clusterResource); VerifyQueueMetrics(b, 4 * Gb, clusterResource); VerifyQueueMetrics(c, 1 * Gb, clusterResource); Org.Mockito.Mockito.Reset(a); Org.Mockito.Mockito.Reset(b); Org.Mockito.Mockito.Reset(c); // Now get both A1, C & B3 to allocate in right order // A = 0/3, B = 4/15, C = 1/6, D=0/6 StubQueueAllocation(a1, clusterResource, node_0, 1 * Gb); StubQueueAllocation(b3, clusterResource, node_0, 2 * Gb); StubQueueAllocation(c, clusterResource, node_0, 2 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); InOrder allocationOrder = Org.Mockito.Mockito.InOrder(a, c, b); allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(c).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); VerifyQueueMetrics(a, 1 * Gb, clusterResource); VerifyQueueMetrics(b, 6 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); Org.Mockito.Mockito.Reset(a); Org.Mockito.Mockito.Reset(b); Org.Mockito.Mockito.Reset(c); // Now verify max-capacity // A = 1/3, B = 6/15, C = 3/6, D=0/6 // Ensure a1 won't alloc above max-cap although it should get // scheduling opportunity now, right after a2 Log.Info("here"); ((ParentQueue)a).SetMaxCapacity(.1f); // a should be capped at 3/30 StubQueueAllocation(a1, clusterResource, node_2, 1 * Gb); // shouldn't be // allocated due // to max-cap StubQueueAllocation(a2, clusterResource, node_2, 2 * Gb); StubQueueAllocation(b3, clusterResource, node_2, 1 * Gb); StubQueueAllocation(b1, clusterResource, node_2, 1 * Gb); StubQueueAllocation(c, clusterResource, node_2, 1 * Gb); root.AssignContainers(clusterResource, node_2, new ResourceLimits(clusterResource )); allocationOrder = Org.Mockito.Mockito.InOrder(a, a2, a1, b, c); allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(a2).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(c).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); VerifyQueueMetrics(a, 3 * Gb, clusterResource); VerifyQueueMetrics(b, 8 * Gb, clusterResource); VerifyQueueMetrics(c, 4 * Gb, clusterResource); Org.Mockito.Mockito.Reset(a); Org.Mockito.Mockito.Reset(b); Org.Mockito.Mockito.Reset(c); }
public virtual void TestSingleLevelQueues() { // Setup queue configs SetupSingleLevelQueues(csConf); IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>(); CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration .Root, queues, queues, TestUtils.spyHook); // Setup some nodes int memoryPerNode = 10; int coresPerNode = 16; int numNodes = 2; FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode * Gb); FiCaSchedulerNode node_1 = TestUtils.GetMockNode("host_1", DefaultRack, 0, memoryPerNode * Gb); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (numNodes * (memoryPerNode * Gb), numNodes * coresPerNode); Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes); // Start testing LeafQueue a = (LeafQueue)queues[A]; LeafQueue b = (LeafQueue)queues[B]; // Simulate B returning a container on node_0 StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 1 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); VerifyQueueMetrics(a, 0 * Gb, clusterResource); VerifyQueueMetrics(b, 1 * Gb, clusterResource); // Now, A should get the scheduling opportunity since A=0G/6G, B=1G/14G StubQueueAllocation(a, clusterResource, node_1, 2 * Gb); StubQueueAllocation(b, clusterResource, node_1, 1 * Gb); root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource )); InOrder allocationOrder = Org.Mockito.Mockito.InOrder(a, b); allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); VerifyQueueMetrics(a, 2 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); // Now, B should get the scheduling opportunity // since A has 2/6G while B has 2/14G StubQueueAllocation(a, clusterResource, node_0, 1 * Gb); StubQueueAllocation(b, clusterResource, node_0, 2 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); allocationOrder = Org.Mockito.Mockito.InOrder(b, a); allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); VerifyQueueMetrics(a, 3 * Gb, clusterResource); VerifyQueueMetrics(b, 4 * Gb, clusterResource); // Now, B should still get the scheduling opportunity // since A has 3/6G while B has 4/14G StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 4 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); allocationOrder = Org.Mockito.Mockito.InOrder(b, a); allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); VerifyQueueMetrics(a, 3 * Gb, clusterResource); VerifyQueueMetrics(b, 8 * Gb, clusterResource); // Now, A should get the scheduling opportunity // since A has 3/6G while B has 8/14G StubQueueAllocation(a, clusterResource, node_1, 1 * Gb); StubQueueAllocation(b, clusterResource, node_1, 1 * Gb); root.AssignContainers(clusterResource, node_1, new ResourceLimits(clusterResource )); allocationOrder = Org.Mockito.Mockito.InOrder(a, b); allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); allocationOrder.Verify(a).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), AnyResourceLimits()); VerifyQueueMetrics(a, 4 * Gb, clusterResource); VerifyQueueMetrics(b, 9 * Gb, clusterResource); }
public virtual void TestSortedQueues() { // Setup queue configs SetupSortedQueues(csConf); IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>(); CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration .Root, queues, queues, TestUtils.spyHook); // Setup some nodes int memoryPerNode = 10; int coresPerNode = 16; int numNodes = 1; FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode * Gb); Org.Mockito.Mockito.DoNothing().When(node_0).ReleaseContainer(Matchers.Any <Container >()); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (numNodes * (memoryPerNode * Gb), numNodes * coresPerNode); Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes); // Start testing CSQueue a = queues[A]; CSQueue b = queues[B]; CSQueue c = queues[C]; CSQueue d = queues[D]; string user_0 = "user_0"; // Stub an App and its containerCompleted FiCaSchedulerApp app_0 = GetMockApplication(0, user_0); Org.Mockito.Mockito.DoReturn(true).When(app_0).ContainerCompleted(Matchers.Any <RMContainer >(), Matchers.Any <ContainerStatus>(), Matchers.Any <RMContainerEventType>()); Priority priority = TestUtils.CreateMockPriority(1); ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer >(); DrainDispatcher drainDispatcher = new DrainDispatcher(); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher >(); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer ); Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher); Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer ); Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher ); Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration ()); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app_0.GetApplicationId (), 1); ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1); Container container = TestUtils.GetMockContainer(containerId, node_0.GetNodeID(), Resources.CreateResource(1 * Gb), priority); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, node_0.GetNodeID (), "user", rmContext); // Assign {1,2,3,4} 1GB containers respectively to queues StubQueueAllocation(a, clusterResource, node_0, 1 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); for (int i = 0; i < 2; i++) { StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 1 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); } for (int i_1 = 0; i_1 < 3; i_1++) { StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 1 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); } for (int i_2 = 0; i_2 < 4; i_2++) { StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 1 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); } VerifyQueueMetrics(a, 1 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 4 * Gb, clusterResource); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); //Release 3 x 1GB containers from D for (int i_3 = 0; i_3 < 3; i_3++) { d.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType .Kill, null, true); } VerifyQueueMetrics(a, 1 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); //reset manually resources on node node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 1 - 2 - 3 - 1) * Gb); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); // Assign 2 x 1GB Containers to A for (int i_4 = 0; i_4 < 2; i_4++) { StubQueueAllocation(a, clusterResource, node_0, 1 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); } VerifyQueueMetrics(a, 3 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); //Release 1GB Container from A a.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType .Kill, null, true); VerifyQueueMetrics(a, 2 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); //reset manually resources on node node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 - 3 - 1) * Gb); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); // Assign 1GB container to B StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 1 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); VerifyQueueMetrics(a, 2 * Gb, clusterResource); VerifyQueueMetrics(b, 3 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); //Release 1GB container resources from B b.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType .Kill, null, true); VerifyQueueMetrics(a, 2 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); //reset manually resources on node node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 - 3 - 1) * Gb); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); // Assign 1GB container to A StubQueueAllocation(a, clusterResource, node_0, 1 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); VerifyQueueMetrics(a, 3 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); // Now do the real test, where B and D request a 1GB container // D should should get the next container if the order is correct StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 1 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 1 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); InOrder allocationOrder = Org.Mockito.Mockito.InOrder(d, b); allocationOrder.Verify(d).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>()); allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>()); VerifyQueueMetrics(a, 3 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 2 * Gb, clusterResource); //D got the container Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); }
// check symlinks given as path arguments are processed correctly with the // follow option /// <exception cref="System.IO.IOException"/> public virtual void ProcessArgumentsOptionFollow() { List <PathData> items = CreateDirectories(); Org.Apache.Hadoop.FS.Shell.Find.Find find = new Org.Apache.Hadoop.FS.Shell.Find.Find (); find.GetOptions().SetFollowLink(true); find.SetConf(conf); TextWriter @out = Org.Mockito.Mockito.Mock <TextWriter>(); find.GetOptions().SetOut(@out); TextWriter err = Org.Mockito.Mockito.Mock <TextWriter>(); find.GetOptions().SetErr(err); Expression expr = Org.Mockito.Mockito.Mock <Expression>(); Org.Mockito.Mockito.When(expr.Apply((PathData)Matchers.Any(), Matchers.AnyInt())) .ThenReturn(Result.Pass); TestFind.FileStatusChecker fsCheck = Org.Mockito.Mockito.Mock <TestFind.FileStatusChecker >(); Expression test = new TestFind.TestExpression(this, expr, fsCheck); find.SetRootExpression(test); find.ProcessArguments(items); InOrder inOrder = Org.Mockito.Mockito.InOrder(expr); inOrder.Verify(expr).SetOptions(find.GetOptions()); inOrder.Verify(expr).Prepare(); inOrder.Verify(expr).Apply(item1, 0); inOrder.Verify(expr).Apply(item1a, 1); inOrder.Verify(expr).Apply(item1aa, 2); inOrder.Verify(expr).Apply(item1b, 1); inOrder.Verify(expr).Apply(item2, 0); inOrder.Verify(expr).Apply(item3, 0); inOrder.Verify(expr).Apply(item4, 0); inOrder.Verify(expr).Apply(item5, 0); inOrder.Verify(expr).Apply(item5a, 1); inOrder.Verify(expr).Apply(item5b, 1); // triggers infinite loop message inOrder.Verify(expr).Apply(item5c, 1); inOrder.Verify(expr).Apply(item5ca, 2); inOrder.Verify(expr).Apply(item5d, 1); inOrder.Verify(expr).Apply(item5ca, 2); // following item5d symlink inOrder.Verify(expr).Apply(item5e, 1); inOrder.Verify(expr).Finish(); Org.Mockito.Mockito.VerifyNoMoreInteractions(expr); InOrder inOrderFsCheck = Org.Mockito.Mockito.InOrder(fsCheck); inOrderFsCheck.Verify(fsCheck).Check(item1.stat); inOrderFsCheck.Verify(fsCheck).Check(item1a.stat); inOrderFsCheck.Verify(fsCheck).Check(item1aa.stat); inOrderFsCheck.Verify(fsCheck).Check(item1b.stat); inOrderFsCheck.Verify(fsCheck).Check(item2.stat); inOrderFsCheck.Verify(fsCheck, Org.Mockito.Mockito.Times(2)).Check(item3.stat); inOrderFsCheck.Verify(fsCheck).Check(item5.stat); inOrderFsCheck.Verify(fsCheck).Check(item1b.stat); inOrderFsCheck.Verify(fsCheck).Check(item5.stat); inOrderFsCheck.Verify(fsCheck).Check(item5c.stat); inOrderFsCheck.Verify(fsCheck).Check(item5ca.stat); inOrderFsCheck.Verify(fsCheck).Check(item5c.stat); inOrderFsCheck.Verify(fsCheck, Org.Mockito.Mockito.Times(2)).Check(item5ca.stat); Org.Mockito.Mockito.VerifyNoMoreInteractions(fsCheck); Org.Mockito.Mockito.VerifyNoMoreInteractions(@out); Org.Mockito.Mockito.Verify(err).WriteLine("Infinite loop ignored: " + item5b.ToString () + " -> " + item5.ToString()); Org.Mockito.Mockito.VerifyNoMoreInteractions(err); }