public virtual void TestNMWebApp() { Context nmContext = new NodeManager.NMContext(null, null, null, null, null); ResourceView resourceView = new _ResourceView_147(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath()); conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath()); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager (conf), dirsHandler); server.Init(conf); server.Start(); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); Dispatcher dispatcher = new AsyncDispatcher(); string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); nmContext.GetApplications()[appId] = app; ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 1); NodeManagerMetrics metrics = Org.Mockito.Mockito.Mock <NodeManagerMetrics>(); NMStateStoreService stateStore = new NMNullStateStoreService(); foreach (ContainerId containerId in new ContainerId[] { container1, container2 }) { // TODO: Use builder utils ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); long currentTime = Runtime.CurrentTimeMillis(); Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234 , user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString ("password"), currentTime); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container = new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics , BuilderUtils.NewContainerTokenIdentifier(containerToken)); nmContext.GetContainers()[containerId] = container; //TODO: Gross hack. Fix in code. ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId (); nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container; WriteContainerLogs(nmContext, containerId, dirsHandler); } }
public virtual void TestUpdateHeartbeatResponseForCleanup() { RMNodeImpl node = GetRunningNode(); NodeId nodeId = node.GetNodeID(); // Expire a container ContainerId completedContainerId = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId (BuilderUtils.NewApplicationId(0, 0), 0), 0); node.Handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId)); NUnit.Framework.Assert.AreEqual(1, node.GetContainersToCleanUp().Count); // Finish an application ApplicationId finishedAppId = BuilderUtils.NewApplicationId(0, 1); node.Handle(new RMNodeCleanAppEvent(nodeId, finishedAppId)); NUnit.Framework.Assert.AreEqual(1, node.GetAppsToCleanup().Count); // Verify status update does not clear containers/apps to cleanup // but updating heartbeat response for cleanup does RMNodeStatusEvent statusEvent = GetMockRMNodeStatusEvent(); node.Handle(statusEvent); NUnit.Framework.Assert.AreEqual(1, node.GetContainersToCleanUp().Count); NUnit.Framework.Assert.AreEqual(1, node.GetAppsToCleanup().Count); NodeHeartbeatResponse hbrsp = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NodeHeartbeatResponse >(); node.UpdateNodeHeartbeatResponseForCleanup(hbrsp); NUnit.Framework.Assert.AreEqual(0, node.GetContainersToCleanUp().Count); NUnit.Framework.Assert.AreEqual(0, node.GetAppsToCleanup().Count); NUnit.Framework.Assert.AreEqual(1, hbrsp.GetContainersToCleanup().Count); NUnit.Framework.Assert.AreEqual(completedContainerId, hbrsp.GetContainersToCleanup ()[0]); NUnit.Framework.Assert.AreEqual(1, hbrsp.GetApplicationsToCleanup().Count); NUnit.Framework.Assert.AreEqual(finishedAppId, hbrsp.GetApplicationsToCleanup()[0 ]); }
public virtual void TestExpiredContainer() { // Start the node node.Handle(new RMNodeStartedEvent(null, null, null)); Org.Mockito.Mockito.Verify(scheduler).Handle(Matchers.Any <NodeAddedSchedulerEvent >()); // Expire a container ContainerId completedContainerId = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId (BuilderUtils.NewApplicationId(0, 0), 0), 0); node.Handle(new RMNodeCleanContainerEvent(null, completedContainerId)); NUnit.Framework.Assert.AreEqual(1, node.GetContainersToCleanUp().Count); // Now verify that scheduler isn't notified of an expired container // by checking number of 'completedContainers' it got in the previous event RMNodeStatusEvent statusEvent = GetMockRMNodeStatusEvent(); ContainerStatus containerStatus = Org.Mockito.Mockito.Mock <ContainerStatus>(); Org.Mockito.Mockito.DoReturn(completedContainerId).When(containerStatus).GetContainerId (); Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatus)). When(statusEvent).GetContainers(); node.Handle(statusEvent); /* Expect the scheduler call handle function 2 times * 1. RMNode status from new to Running, handle the add_node event * 2. handle the node update event */ Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(2)).Handle(Matchers.Any <NodeUpdateSchedulerEvent>()); }
/// <exception cref="System.IO.IOException"/> private Dictionary <string, string> AddAppContainers(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app) { Dispatcher dispatcher = new AsyncDispatcher(); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app.GetAppId (), 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container1 = new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId (), 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container2 = new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId (), 2); nmContext.GetContainers()[container1.GetContainerId()] = container1; nmContext.GetContainers()[container2.GetContainerId()] = container2; app.GetContainers()[container1.GetContainerId()] = container1; app.GetContainers()[container2.GetContainerId()] = container2; Dictionary <string, string> hash = new Dictionary <string, string>(); hash[container1.GetContainerId().ToString()] = container1.GetContainerId().ToString (); hash[container2.GetContainerId().ToString()] = container2.GetContainerId().ToString (); return(hash); }
public virtual void TestStatusChange() { //Start the node node.Handle(new RMNodeStartedEvent(null, null, null)); //Add info to the queue first node.SetNextHeartBeat(false); ContainerId completedContainerId1 = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId (BuilderUtils.NewApplicationId(0, 0), 0), 0); ContainerId completedContainerId2 = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId (BuilderUtils.NewApplicationId(1, 1), 1), 1); RMNodeStatusEvent statusEvent1 = GetMockRMNodeStatusEvent(); RMNodeStatusEvent statusEvent2 = GetMockRMNodeStatusEvent(); ContainerStatus containerStatus1 = Org.Mockito.Mockito.Mock <ContainerStatus>(); ContainerStatus containerStatus2 = Org.Mockito.Mockito.Mock <ContainerStatus>(); Org.Mockito.Mockito.DoReturn(completedContainerId1).When(containerStatus1).GetContainerId (); Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatus1)) .When(statusEvent1).GetContainers(); Org.Mockito.Mockito.DoReturn(completedContainerId2).When(containerStatus2).GetContainerId (); Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatus2)) .When(statusEvent2).GetContainers(); Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(1)).Handle(Matchers.Any <NodeUpdateSchedulerEvent>()); node.Handle(statusEvent1); node.Handle(statusEvent2); Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(1)).Handle(Matchers.Any <NodeUpdateSchedulerEvent>()); NUnit.Framework.Assert.AreEqual(2, node.GetQueueSize()); node.Handle(new RMNodeEvent(node.GetNodeID(), RMNodeEventType.Expire)); NUnit.Framework.Assert.AreEqual(0, node.GetQueueSize()); }
private static AppSchedulingInfo MockApp(string user) { AppSchedulingInfo app = Org.Mockito.Mockito.Mock <AppSchedulingInfo>(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); ApplicationId appId = BuilderUtils.NewApplicationId(1, 1); ApplicationAttemptId id = BuilderUtils.NewApplicationAttemptId(appId, 1); Org.Mockito.Mockito.When(app.GetApplicationAttemptId()).ThenReturn(id); return(app); }
/// <exception cref="System.IO.IOException"/> private void VerifyTokenWithTamperedID(Configuration conf, TestClientToAMTokens.CustomAM am, Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token) { // Malicious user, messes with appId UserGroupInformation ugi = UserGroupInformation.CreateRemoteUser("me"); ClientToAMTokenIdentifier maliciousID = new ClientToAMTokenIdentifier(BuilderUtils .NewApplicationAttemptId(BuilderUtils.NewApplicationId(am.appAttemptId.GetApplicationId ().GetClusterTimestamp(), 42), 43), UserGroupInformation.GetCurrentUser().GetShortUserName ()); VerifyTamperedToken(conf, am, token, ugi, maliciousID); }
public virtual void Setup() { mockDelService = Org.Mockito.Mockito.Mock <DeletionService>(); conf = new YarnConfiguration(); dispatcher = CreateDispatcher(conf); appEventHandler = Org.Mockito.Mockito.Mock <EventHandler>(); dispatcher.Register(typeof(ApplicationEventType), appEventHandler); appId = BuilderUtils.NewApplicationId(1234, 1); appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1); container11 = BuilderUtils.NewContainerId(appAttemptId, 1); dirsHandler = new LocalDirsHandlerService(); }
/// <exception cref="System.Exception"/> public virtual void TestContainerUpdate() { //Start the node node.Handle(new RMNodeStartedEvent(null, null, null)); NodeId nodeId = BuilderUtils.NewNodeId("localhost:1", 1); RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null ); node2.Handle(new RMNodeStartedEvent(null, null, null)); ContainerId completedContainerIdFromNode1 = BuilderUtils.NewContainerId(BuilderUtils .NewApplicationAttemptId(BuilderUtils.NewApplicationId(0, 0), 0), 0); ContainerId completedContainerIdFromNode2_1 = BuilderUtils.NewContainerId(BuilderUtils .NewApplicationAttemptId(BuilderUtils.NewApplicationId(1, 1), 1), 1); ContainerId completedContainerIdFromNode2_2 = BuilderUtils.NewContainerId(BuilderUtils .NewApplicationAttemptId(BuilderUtils.NewApplicationId(1, 1), 1), 2); RMNodeStatusEvent statusEventFromNode1 = GetMockRMNodeStatusEvent(); RMNodeStatusEvent statusEventFromNode2_1 = GetMockRMNodeStatusEvent(); RMNodeStatusEvent statusEventFromNode2_2 = GetMockRMNodeStatusEvent(); ContainerStatus containerStatusFromNode1 = Org.Mockito.Mockito.Mock <ContainerStatus >(); ContainerStatus containerStatusFromNode2_1 = Org.Mockito.Mockito.Mock <ContainerStatus >(); ContainerStatus containerStatusFromNode2_2 = Org.Mockito.Mockito.Mock <ContainerStatus >(); Org.Mockito.Mockito.DoReturn(completedContainerIdFromNode1).When(containerStatusFromNode1 ).GetContainerId(); Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatusFromNode1 )).When(statusEventFromNode1).GetContainers(); node.Handle(statusEventFromNode1); NUnit.Framework.Assert.AreEqual(1, completedContainers.Count); NUnit.Framework.Assert.AreEqual(completedContainerIdFromNode1, completedContainers [0].GetContainerId()); completedContainers.Clear(); Org.Mockito.Mockito.DoReturn(completedContainerIdFromNode2_1).When(containerStatusFromNode2_1 ).GetContainerId(); Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatusFromNode2_1 )).When(statusEventFromNode2_1).GetContainers(); Org.Mockito.Mockito.DoReturn(completedContainerIdFromNode2_2).When(containerStatusFromNode2_2 ).GetContainerId(); Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatusFromNode2_2 )).When(statusEventFromNode2_2).GetContainers(); node2.SetNextHeartBeat(false); node2.Handle(statusEventFromNode2_1); node2.SetNextHeartBeat(true); node2.Handle(statusEventFromNode2_2); NUnit.Framework.Assert.AreEqual(2, completedContainers.Count); NUnit.Framework.Assert.AreEqual(completedContainerIdFromNode2_1, completedContainers [0].GetContainerId()); NUnit.Framework.Assert.AreEqual(completedContainerIdFromNode2_2, completedContainers [1].GetContainerId()); }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void TestContainerLogFile() { FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir" ).GetAbsoluteFile(); string logdirwithFile = absLogDir.ToURI().ToString(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile); conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f); LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService(); dirsHandler.Init(conf); NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler , new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(clusterTimeStamp, 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1); nmContext.GetApplications()[appId] = app; MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[containerId] = container; FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir (appId.ToString(), containerId.ToString())); containerLogDir.Mkdirs(); string fileName = "fileName"; FilePath containerLogFile = new FilePath(containerLogDir, fileName); containerLogFile.CreateNewFile(); FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user , nmContext); NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI() .ToString()); FileUtil.FullyDelete(absLogDir); }
CreateMockedContainer(ApplicationId appId, int containerId) { ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId cId = BuilderUtils.NewContainerId(appAttemptId, containerId); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container c = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); Org.Mockito.Mockito.When(c.GetContainerId()).ThenReturn(cId); ContainerLaunchContext launchContext = Org.Mockito.Mockito.Mock <ContainerLaunchContext >(); Org.Mockito.Mockito.When(c.GetLaunchContext()).ThenReturn(launchContext); Org.Mockito.Mockito.When(launchContext.GetApplicationACLs()).ThenReturn(new Dictionary <ApplicationAccessType, string>()); return(c); }
/// <exception cref="System.Exception"/> public virtual void TestAppAttemptMetrics() { AsyncDispatcher dispatcher = new InlineDispatcher(); FifoScheduler scheduler = new FifoScheduler(); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null, null, null, null, scheduler); ((RMContextImpl)rmContext).SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher >()); Configuration conf = new Configuration(); scheduler.SetRMContext(rmContext); scheduler.Init(conf); scheduler.Start(); scheduler.Reinitialize(conf, rmContext); QueueMetrics metrics = scheduler.GetRootQueueMetrics(); int beforeAppsSubmitted = metrics.GetAppsSubmitted(); ApplicationId appId = BuilderUtils.NewApplicationId(200, 1); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user"); scheduler.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false ); scheduler.Handle(attemptEvent); appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 2); SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false ); scheduler.Handle(attemptEvent2); int afterAppsSubmitted = metrics.GetAppsSubmitted(); NUnit.Framework.Assert.AreEqual(1, afterAppsSubmitted - beforeAppsSubmitted); scheduler.Stop(); }
public virtual void TestBlackListNodes() { Configuration conf = new Configuration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); rm.Start(); FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler(); string host = "127.0.0.1"; RMNode node = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, host); fs.Handle(new NodeAddedSchedulerEvent(node)); ApplicationId appId = BuilderUtils.NewApplicationId(100, 1); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); CreateMockRMApp(appAttemptId, rm.GetRMContext()); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "default", "user"); fs.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false ); fs.Handle(attemptEvent); // Verify the blacklist can be updated independent of requesting containers fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections .EmptyList <ContainerId>(), Sharpen.Collections.SingletonList(host), null); NUnit.Framework.Assert.IsTrue(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted (host)); fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections .EmptyList <ContainerId>(), null, Sharpen.Collections.SingletonList(host)); NUnit.Framework.Assert.IsFalse(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted (host)); rm.Stop(); }
public virtual void TestSortedQueues() { // Setup queue configs SetupSortedQueues(csConf); IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>(); CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration .Root, queues, queues, TestUtils.spyHook); // Setup some nodes int memoryPerNode = 10; int coresPerNode = 16; int numNodes = 1; FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode * Gb); Org.Mockito.Mockito.DoNothing().When(node_0).ReleaseContainer(Matchers.Any <Container >()); Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource (numNodes * (memoryPerNode * Gb), numNodes * coresPerNode); Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes); // Start testing CSQueue a = queues[A]; CSQueue b = queues[B]; CSQueue c = queues[C]; CSQueue d = queues[D]; string user_0 = "user_0"; // Stub an App and its containerCompleted FiCaSchedulerApp app_0 = GetMockApplication(0, user_0); Org.Mockito.Mockito.DoReturn(true).When(app_0).ContainerCompleted(Matchers.Any <RMContainer >(), Matchers.Any <ContainerStatus>(), Matchers.Any <RMContainerEventType>()); Priority priority = TestUtils.CreateMockPriority(1); ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer >(); DrainDispatcher drainDispatcher = new DrainDispatcher(); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher >(); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer ); Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher); Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer ); Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher ); Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration ()); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app_0.GetApplicationId (), 1); ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1); Container container = TestUtils.GetMockContainer(containerId, node_0.GetNodeID(), Resources.CreateResource(1 * Gb), priority); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, node_0.GetNodeID (), "user", rmContext); // Assign {1,2,3,4} 1GB containers respectively to queues StubQueueAllocation(a, clusterResource, node_0, 1 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); for (int i = 0; i < 2; i++) { StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 1 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); } for (int i_1 = 0; i_1 < 3; i_1++) { StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 1 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); } for (int i_2 = 0; i_2 < 4; i_2++) { StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 1 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); } VerifyQueueMetrics(a, 1 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 4 * Gb, clusterResource); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); //Release 3 x 1GB containers from D for (int i_3 = 0; i_3 < 3; i_3++) { d.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType .Kill, null, true); } VerifyQueueMetrics(a, 1 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); //reset manually resources on node node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 1 - 2 - 3 - 1) * Gb); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); // Assign 2 x 1GB Containers to A for (int i_4 = 0; i_4 < 2; i_4++) { StubQueueAllocation(a, clusterResource, node_0, 1 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); } VerifyQueueMetrics(a, 3 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); //Release 1GB Container from A a.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType .Kill, null, true); VerifyQueueMetrics(a, 2 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); //reset manually resources on node node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 - 3 - 1) * Gb); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); // Assign 1GB container to B StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 1 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); VerifyQueueMetrics(a, 2 * Gb, clusterResource); VerifyQueueMetrics(b, 3 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); //Release 1GB container resources from B b.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType .Kill, null, true); VerifyQueueMetrics(a, 2 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); //reset manually resources on node node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 - 3 - 1) * Gb); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); // Assign 1GB container to A StubQueueAllocation(a, clusterResource, node_0, 1 * Gb); StubQueueAllocation(b, clusterResource, node_0, 0 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 0 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); VerifyQueueMetrics(a, 3 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 1 * Gb, clusterResource); Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); // Now do the real test, where B and D request a 1GB container // D should should get the next container if the order is correct StubQueueAllocation(a, clusterResource, node_0, 0 * Gb); StubQueueAllocation(b, clusterResource, node_0, 1 * Gb); StubQueueAllocation(c, clusterResource, node_0, 0 * Gb); StubQueueAllocation(d, clusterResource, node_0, 1 * Gb); root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource )); InOrder allocationOrder = Org.Mockito.Mockito.InOrder(d, b); allocationOrder.Verify(d).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>()); allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any <FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>()); VerifyQueueMetrics(a, 3 * Gb, clusterResource); VerifyQueueMetrics(b, 2 * Gb, clusterResource); VerifyQueueMetrics(c, 3 * Gb, clusterResource); VerifyQueueMetrics(d, 2 * Gb, clusterResource); //D got the container Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint()); }
/// <exception cref="System.Exception"/> public virtual void TestHeadroom() { Configuration conf = new Configuration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); rm.Start(); FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler(); // Add a node RMNode n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, "127.0.0.2" ); fs.Handle(new NodeAddedSchedulerEvent(n1)); // Add two applications ApplicationId appId1 = BuilderUtils.NewApplicationId(100, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1, 1); CreateMockRMApp(appAttemptId1, rm.GetRMContext()); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user"); fs.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false ); fs.Handle(attemptEvent); ApplicationId appId2 = BuilderUtils.NewApplicationId(200, 2); ApplicationAttemptId appAttemptId2 = BuilderUtils.NewApplicationAttemptId(appId2, 1); CreateMockRMApp(appAttemptId2, rm.GetRMContext()); SchedulerEvent appEvent2 = new AppAddedSchedulerEvent(appId2, "queue", "user"); fs.Handle(appEvent2); SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false ); fs.Handle(attemptEvent2); IList <ContainerId> emptyId = new AList <ContainerId>(); IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>(); // Set up resource requests // Ask for a 1 GB container for app 1 IList <ResourceRequest> ask1 = new AList <ResourceRequest>(); ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest .Any, BuilderUtils.NewResource(Gb, 1), 1)); fs.Allocate(appAttemptId1, ask1, emptyId, null, null); // Ask for a 2 GB container for app 2 IList <ResourceRequest> ask2 = new AList <ResourceRequest>(); ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest .Any, BuilderUtils.NewResource(2 * Gb, 1), 1)); fs.Allocate(appAttemptId2, ask2, emptyId, null, null); // Trigger container assignment fs.Handle(new NodeUpdateSchedulerEvent(n1)); // Get the allocation for the applications and verify headroom Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation1.GetResourceLimit ().GetMemory()); Allocation allocation2 = fs.Allocate(appAttemptId2, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation2.GetResourceLimit ().GetMemory()); rm.Stop(); }
/// <exception cref="System.Exception"/> public virtual void TestBlackListNodes() { Configuration conf = new Configuration(); conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler )); MockRM rm = new MockRM(conf); rm.Start(); FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler(); int rack_num_0 = 0; int rack_num_1 = 1; // Add 4 nodes in 2 racks // host_0_0 in rack0 string host_0_0 = "127.0.0.1"; RMNode n1 = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_0 ); fs.Handle(new NodeAddedSchedulerEvent(n1)); // host_0_1 in rack0 string host_0_1 = "127.0.0.2"; RMNode n2 = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_1 ); fs.Handle(new NodeAddedSchedulerEvent(n2)); // host_1_0 in rack1 string host_1_0 = "127.0.0.3"; RMNode n3 = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_0 ); fs.Handle(new NodeAddedSchedulerEvent(n3)); // host_1_1 in rack1 string host_1_1 = "127.0.0.4"; RMNode n4 = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_1 ); fs.Handle(new NodeAddedSchedulerEvent(n4)); // Add one application ApplicationId appId1 = BuilderUtils.NewApplicationId(100, 1); ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1, 1); CreateMockRMApp(appAttemptId1, rm.GetRMContext()); SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user"); fs.Handle(appEvent); SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false ); fs.Handle(attemptEvent); IList <ContainerId> emptyId = new AList <ContainerId>(); IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>(); // Allow rack-locality for rack_1, but blacklist host_1_0 // Set up resource requests // Ask for a 1 GB container for app 1 IList <ResourceRequest> ask1 = new AList <ResourceRequest>(); ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), "rack1" , BuilderUtils.NewResource(Gb, 1), 1)); ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest .Any, BuilderUtils.NewResource(Gb, 1), 1)); fs.Allocate(appAttemptId1, ask1, emptyId, Sharpen.Collections.SingletonList(host_1_0 ), null); // Trigger container assignment fs.Handle(new NodeUpdateSchedulerEvent(n3)); // Get the allocation for the application and verify no allocation on blacklist node Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation1", 0, allocation1.GetContainers().Count ); // verify host_1_1 can get allocated as not in blacklist fs.Handle(new NodeUpdateSchedulerEvent(n4)); Allocation allocation2 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation2", 1, allocation2.GetContainers().Count ); IList <Container> containerList = allocation2.GetContainers(); foreach (Container container in containerList) { NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container.GetNodeId (), n4.GetNodeID()); } // Ask for a 1 GB container again for app 1 IList <ResourceRequest> ask2 = new AList <ResourceRequest>(); // this time, rack0 is also in blacklist, so only host_1_1 is available to // be assigned ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest .Any, BuilderUtils.NewResource(Gb, 1), 1)); fs.Allocate(appAttemptId1, ask2, emptyId, Sharpen.Collections.SingletonList("rack0" ), null); // verify n1 is not qualified to be allocated fs.Handle(new NodeUpdateSchedulerEvent(n1)); Allocation allocation3 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation3", 0, allocation3.GetContainers().Count ); // verify n2 is not qualified to be allocated fs.Handle(new NodeUpdateSchedulerEvent(n2)); Allocation allocation4 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation4", 0, allocation4.GetContainers().Count ); // verify n3 is not qualified to be allocated fs.Handle(new NodeUpdateSchedulerEvent(n3)); Allocation allocation5 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation5", 0, allocation5.GetContainers().Count ); fs.Handle(new NodeUpdateSchedulerEvent(n4)); Allocation allocation6 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null ); NUnit.Framework.Assert.AreEqual("allocation6", 1, allocation6.GetContainers().Count ); containerList = allocation6.GetContainers(); foreach (Container container_1 in containerList) { NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container_1.GetNodeId (), n4.GetNodeID()); } rm.Stop(); }
public virtual void TestReleaseWhileRunning() { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock <EventHandler>(); EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>(); drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler); drainDispatcher.Register(typeof(RMNodeEventType), generic); drainDispatcher.Init(new YarnConfiguration()); drainDispatcher.Start(); NodeId nodeId = BuilderUtils.NewNodeId("host", 3425); ApplicationId appId = BuilderUtils.NewApplicationId(1, 1); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1); ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer >(); Resource resource = BuilderUtils.NewResource(512, 1); Priority priority = BuilderUtils.NewPriority(5); Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465", resource, priority, null); ConcurrentMap <ApplicationId, RMApp> rmApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap <ApplicationId, RMApp>()); RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>(); Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any ())).ThenReturn(null); Org.Mockito.Mockito.DoReturn(rmApp).When(rmApps)[(ApplicationId)Matchers.Any()]; RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher >(); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher); Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer ); Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer ); Org.Mockito.Mockito.When(rmContext.GetRMApps()).ThenReturn(rmApps); Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher ); Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration ()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user" , rmContext); NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState()); NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource()); NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode()); NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority()); Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>()); Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>( ), Matchers.AnyLong()); rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start)); drainDispatcher.Await(); NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState( )); rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired )); drainDispatcher.Await(); NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState() ); rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched )); drainDispatcher.Await(); NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState()); NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user" , rmContainer.GetLogURL()); // In RUNNING state. Verify RELEASED and associated actions. Org.Mockito.Mockito.Reset(appAttemptEventHandler); ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId , SchedulerUtils.ReleasedContainer); rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType .Released)); drainDispatcher.Await(); NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState() ); NUnit.Framework.Assert.AreEqual(SchedulerUtils.ReleasedContainer, rmContainer.GetDiagnosticsInfo ()); NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, rmContainer.GetContainerExitStatus ()); NUnit.Framework.Assert.AreEqual(ContainerState.Complete, rmContainer.GetContainerState ()); Org.Mockito.Mockito.Verify(writer).ContainerFinished(Matchers.Any <RMContainer>()); Org.Mockito.Mockito.Verify(publisher).ContainerFinished(Matchers.Any <RMContainer> (), Matchers.AnyLong()); ArgumentCaptor <RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor.ForClass <RMAppAttemptContainerFinishedEvent>(); Org.Mockito.Mockito.Verify(appAttemptEventHandler).Handle(captor.Capture()); RMAppAttemptContainerFinishedEvent cfEvent = captor.GetValue(); NUnit.Framework.Assert.AreEqual(appAttemptId, cfEvent.GetApplicationAttemptId()); NUnit.Framework.Assert.AreEqual(containerStatus, cfEvent.GetContainerStatus()); NUnit.Framework.Assert.AreEqual(RMAppAttemptEventType.ContainerFinished, cfEvent. GetType()); // In RELEASED state. A FINIHSED event may come in. rmContainer.Handle(new RMContainerFinishedEvent(containerId, SchedulerUtils.CreateAbnormalContainerStatus (containerId, "FinishedContainer"), RMContainerEventType.Finished)); NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState() ); }
public virtual void TestExpireWhileRunning() { DrainDispatcher drainDispatcher = new DrainDispatcher(); EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock <EventHandler>(); EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>(); drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler); drainDispatcher.Register(typeof(RMNodeEventType), generic); drainDispatcher.Init(new YarnConfiguration()); drainDispatcher.Start(); NodeId nodeId = BuilderUtils.NewNodeId("host", 3425); ApplicationId appId = BuilderUtils.NewApplicationId(1, 1); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1); ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer >(); Resource resource = BuilderUtils.NewResource(512, 1); Priority priority = BuilderUtils.NewPriority(5); Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465", resource, priority, null); RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter >(); SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher >(); RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>(); Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher); Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer ); Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer ); Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher ); Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration ()); RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user" , rmContext); NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState()); NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource()); NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode()); NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority()); Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>()); Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>( ), Matchers.AnyLong()); rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start)); drainDispatcher.Await(); NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState( )); rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired )); drainDispatcher.Await(); NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState() ); rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched )); drainDispatcher.Await(); NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState()); NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user" , rmContainer.GetLogURL()); // In RUNNING state. Verify EXPIRE and associated actions. Org.Mockito.Mockito.Reset(appAttemptEventHandler); ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId , SchedulerUtils.ExpiredContainer); rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType .Expire)); drainDispatcher.Await(); NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState()); Org.Mockito.Mockito.Verify(writer, Org.Mockito.Mockito.Never()).ContainerFinished (Matchers.Any <RMContainer>()); Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Never()).ContainerFinished (Matchers.Any <RMContainer>(), Matchers.AnyLong()); }
/// <exception cref="System.IO.IOException"/> public virtual void TestContainerLogPageAccess() { // SecureIOUtils require Native IO to be enabled. This test will run // only if it is enabled. Assume.AssumeTrue(NativeIO.IsAvailable()); string user = "******" + Runtime.CurrentTimeMillis(); FilePath absLogDir = null; FilePath appDir = null; FilePath containerDir = null; FilePath syslog = null; try { // target log directory absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir") .GetAbsoluteFile(); absLogDir.Mkdir(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString()); conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos"); UserGroupInformation.SetConfiguration(conf); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); // Making sure that application returns a random user. This is required // for SecureIOUtils' file owner check. Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); // Testing secure read access for log files // Creating application and container directory and syslog file. appDir = new FilePath(absLogDir, appId.ToString()); appDir.Mkdir(); containerDir = new FilePath(appDir, container1.ToString()); containerDir.Mkdir(); syslog = new FilePath(containerDir, "syslog"); syslog.CreateNewFile(); BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog) ); @out.Write(Sharpen.Runtime.GetBytesForString("Log file Content")); @out.Close(); Context context = Org.Mockito.Mockito.Mock <Context>(); ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application > appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); appMap[appId] = app; Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap); ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container >(); Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers); Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler); MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); context.GetContainers()[container1] = container; ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock (context); IDictionary <string, string> @params = new Dictionary <string, string>(); @params[YarnWebParams.ContainerId] = container1.ToString(); @params[YarnWebParams.ContainerLogType] = "syslog"; Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof( ContainerLogsPage), cLogsBlock, @params, (Module[])null); PrintWriter spyPw = WebAppTests.GetPrintWriter(injector); Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '" + user + "' doesn't own requested log file : syslog"); } finally { if (syslog != null) { syslog.Delete(); } if (containerDir != null) { containerDir.Delete(); } if (appDir != null) { appDir.Delete(); } if (absLogDir != null) { absLogDir.Delete(); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/> public virtual void TestContainerLogDirs() { FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir" ).GetAbsoluteFile(); string logdirwithFile = absLogDir.ToURI().ToString(); Configuration conf = new Configuration(); conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile); NodeHealthCheckerService healthChecker = new NodeHealthCheckerService(); healthChecker.Init(conf); LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler(); NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler , new ApplicationACLsManager(conf), new NMNullStateStoreService()); // Add an application and the corresponding containers RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(conf); string user = "******"; long clusterTimeStamp = 1234; ApplicationId appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp , 1); Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application >(); Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user); Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId); ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1 ); ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId , 0); nmContext.GetApplications()[appId] = app; MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1); container.SetState(ContainerState.Running); nmContext.GetContainers()[container1] = container; IList <FilePath> files = null; files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext); NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:"))); // After container is completed, it is removed from nmContext Sharpen.Collections.Remove(nmContext.GetContainers(), container1); NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]); files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext); NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:"))); // Create a new context to check if correct container log dirs are fetched // on full disk. LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler ); // good log dirs are empty and nm log dir is in the full log dir list. Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList <string>()); Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays .AsList(new string[] { absLogDir.GetAbsolutePath() })); nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager (conf), new NMNullStateStoreService()); nmContext.GetApplications()[appId] = app; container.SetState(ContainerState.Running); nmContext.GetContainers()[container1] = container; IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext ); FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1); NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir)); }