コード例 #1
0
        public virtual void TestNMWebApp()
        {
            Context       nmContext    = new NodeManager.NMContext(null, null, null, null, null);
            ResourceView  resourceView = new _ResourceView_147();
            Configuration conf         = new Configuration();

            conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
            conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
            NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();

            healthChecker.Init(conf);
            LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
            WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
                                                 (conf), dirsHandler);

            server.Init(conf);
            server.Start();
            // Add an application and the corresponding containers
            RecordFactory recordFactory    = RecordFactoryProvider.GetRecordFactory(conf);
            Dispatcher    dispatcher       = new AsyncDispatcher();
            string        user             = "******";
            long          clusterTimeStamp = 1234;
            ApplicationId appId            = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
                                                                           , 1);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                >();
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
            nmContext.GetApplications()[appId] = app;
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                 , 0);
            ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                 , 1);
            NodeManagerMetrics  metrics    = Org.Mockito.Mockito.Mock <NodeManagerMetrics>();
            NMStateStoreService stateStore = new NMNullStateStoreService();

            foreach (ContainerId containerId in new ContainerId[] { container1, container2 })
            {
                // TODO: Use builder utils
                ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                        >();
                long  currentTime    = Runtime.CurrentTimeMillis();
                Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234
                                                                      , user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString
                                                                          ("password"), currentTime);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
                    = new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics
                                             , BuilderUtils.NewContainerTokenIdentifier(containerToken));
                nmContext.GetContainers()[containerId] = container;
                //TODO: Gross hack. Fix in code.
                ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
                                                  ();
                nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container;
                WriteContainerLogs(nmContext, containerId, dirsHandler);
            }
        }
コード例 #2
0
        public virtual void TestUpdateHeartbeatResponseForCleanup()
        {
            RMNodeImpl node   = GetRunningNode();
            NodeId     nodeId = node.GetNodeID();
            // Expire a container
            ContainerId completedContainerId = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId
                                                                               (BuilderUtils.NewApplicationId(0, 0), 0), 0);

            node.Handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId));
            NUnit.Framework.Assert.AreEqual(1, node.GetContainersToCleanUp().Count);
            // Finish an application
            ApplicationId finishedAppId = BuilderUtils.NewApplicationId(0, 1);

            node.Handle(new RMNodeCleanAppEvent(nodeId, finishedAppId));
            NUnit.Framework.Assert.AreEqual(1, node.GetAppsToCleanup().Count);
            // Verify status update does not clear containers/apps to cleanup
            // but updating heartbeat response for cleanup does
            RMNodeStatusEvent statusEvent = GetMockRMNodeStatusEvent();

            node.Handle(statusEvent);
            NUnit.Framework.Assert.AreEqual(1, node.GetContainersToCleanUp().Count);
            NUnit.Framework.Assert.AreEqual(1, node.GetAppsToCleanup().Count);
            NodeHeartbeatResponse hbrsp = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NodeHeartbeatResponse
                                                                                         >();

            node.UpdateNodeHeartbeatResponseForCleanup(hbrsp);
            NUnit.Framework.Assert.AreEqual(0, node.GetContainersToCleanUp().Count);
            NUnit.Framework.Assert.AreEqual(0, node.GetAppsToCleanup().Count);
            NUnit.Framework.Assert.AreEqual(1, hbrsp.GetContainersToCleanup().Count);
            NUnit.Framework.Assert.AreEqual(completedContainerId, hbrsp.GetContainersToCleanup
                                                ()[0]);
            NUnit.Framework.Assert.AreEqual(1, hbrsp.GetApplicationsToCleanup().Count);
            NUnit.Framework.Assert.AreEqual(finishedAppId, hbrsp.GetApplicationsToCleanup()[0
                                            ]);
        }
コード例 #3
0
        public virtual void TestExpiredContainer()
        {
            // Start the node
            node.Handle(new RMNodeStartedEvent(null, null, null));
            Org.Mockito.Mockito.Verify(scheduler).Handle(Matchers.Any <NodeAddedSchedulerEvent
                                                                       >());
            // Expire a container
            ContainerId completedContainerId = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId
                                                                               (BuilderUtils.NewApplicationId(0, 0), 0), 0);

            node.Handle(new RMNodeCleanContainerEvent(null, completedContainerId));
            NUnit.Framework.Assert.AreEqual(1, node.GetContainersToCleanUp().Count);
            // Now verify that scheduler isn't notified of an expired container
            // by checking number of 'completedContainers' it got in the previous event
            RMNodeStatusEvent statusEvent     = GetMockRMNodeStatusEvent();
            ContainerStatus   containerStatus = Org.Mockito.Mockito.Mock <ContainerStatus>();

            Org.Mockito.Mockito.DoReturn(completedContainerId).When(containerStatus).GetContainerId
                ();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatus)).
            When(statusEvent).GetContainers();
            node.Handle(statusEvent);

            /* Expect the scheduler call handle function 2 times
             * 1. RMNode status from new to Running, handle the add_node event
             * 2. handle the node update event
             */
            Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(2)).Handle(Matchers.Any
                                                                                       <NodeUpdateSchedulerEvent>());
        }
コード例 #4
0
        /// <exception cref="System.IO.IOException"/>
        private Dictionary <string, string> AddAppContainers(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                             app)
        {
            Dispatcher           dispatcher   = new AsyncDispatcher();
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app.GetAppId
                                                                                         (), 1);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container1
                = new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId
                                        (), 1);
            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container2
                = new MockContainer(appAttemptId, dispatcher, conf, app.GetUser(), app.GetAppId
                                        (), 2);
            nmContext.GetContainers()[container1.GetContainerId()] = container1;
            nmContext.GetContainers()[container2.GetContainerId()] = container2;
            app.GetContainers()[container1.GetContainerId()]       = container1;
            app.GetContainers()[container2.GetContainerId()]       = container2;
            Dictionary <string, string> hash = new Dictionary <string, string>();

            hash[container1.GetContainerId().ToString()] = container1.GetContainerId().ToString
                                                               ();
            hash[container2.GetContainerId().ToString()] = container2.GetContainerId().ToString
                                                               ();
            return(hash);
        }
コード例 #5
0
        public virtual void TestStatusChange()
        {
            //Start the node
            node.Handle(new RMNodeStartedEvent(null, null, null));
            //Add info to the queue first
            node.SetNextHeartBeat(false);
            ContainerId completedContainerId1 = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId
                                                                                (BuilderUtils.NewApplicationId(0, 0), 0), 0);
            ContainerId completedContainerId2 = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId
                                                                                (BuilderUtils.NewApplicationId(1, 1), 1), 1);
            RMNodeStatusEvent statusEvent1     = GetMockRMNodeStatusEvent();
            RMNodeStatusEvent statusEvent2     = GetMockRMNodeStatusEvent();
            ContainerStatus   containerStatus1 = Org.Mockito.Mockito.Mock <ContainerStatus>();
            ContainerStatus   containerStatus2 = Org.Mockito.Mockito.Mock <ContainerStatus>();

            Org.Mockito.Mockito.DoReturn(completedContainerId1).When(containerStatus1).GetContainerId
                ();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatus1))
            .When(statusEvent1).GetContainers();
            Org.Mockito.Mockito.DoReturn(completedContainerId2).When(containerStatus2).GetContainerId
                ();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatus2))
            .When(statusEvent2).GetContainers();
            Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(1)).Handle(Matchers.Any
                                                                                       <NodeUpdateSchedulerEvent>());
            node.Handle(statusEvent1);
            node.Handle(statusEvent2);
            Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(1)).Handle(Matchers.Any
                                                                                       <NodeUpdateSchedulerEvent>());
            NUnit.Framework.Assert.AreEqual(2, node.GetQueueSize());
            node.Handle(new RMNodeEvent(node.GetNodeID(), RMNodeEventType.Expire));
            NUnit.Framework.Assert.AreEqual(0, node.GetQueueSize());
        }
コード例 #6
0
        private static AppSchedulingInfo MockApp(string user)
        {
            AppSchedulingInfo app = Org.Mockito.Mockito.Mock <AppSchedulingInfo>();

            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            ApplicationId        appId = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId id    = BuilderUtils.NewApplicationAttemptId(appId, 1);

            Org.Mockito.Mockito.When(app.GetApplicationAttemptId()).ThenReturn(id);
            return(app);
        }
コード例 #7
0
        /// <exception cref="System.IO.IOException"/>
        private void VerifyTokenWithTamperedID(Configuration conf, TestClientToAMTokens.CustomAM
                                               am, Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token)
        {
            // Malicious user, messes with appId
            UserGroupInformation      ugi         = UserGroupInformation.CreateRemoteUser("me");
            ClientToAMTokenIdentifier maliciousID = new ClientToAMTokenIdentifier(BuilderUtils
                                                                                  .NewApplicationAttemptId(BuilderUtils.NewApplicationId(am.appAttemptId.GetApplicationId
                                                                                                                                             ().GetClusterTimestamp(), 42), 43), UserGroupInformation.GetCurrentUser().GetShortUserName
                                                                                      ());

            VerifyTamperedToken(conf, am, token, ugi, maliciousID);
        }
コード例 #8
0
 public virtual void Setup()
 {
     mockDelService  = Org.Mockito.Mockito.Mock <DeletionService>();
     conf            = new YarnConfiguration();
     dispatcher      = CreateDispatcher(conf);
     appEventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
     dispatcher.Register(typeof(ApplicationEventType), appEventHandler);
     appId        = BuilderUtils.NewApplicationId(1234, 1);
     appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1);
     container11  = BuilderUtils.NewContainerId(appAttemptId, 1);
     dirsHandler  = new LocalDirsHandlerService();
 }
コード例 #9
0
        /// <exception cref="System.Exception"/>
        public virtual void TestContainerUpdate()
        {
            //Start the node
            node.Handle(new RMNodeStartedEvent(null, null, null));
            NodeId     nodeId = BuilderUtils.NewNodeId("localhost:1", 1);
            RMNodeImpl node2  = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null
                                               );

            node2.Handle(new RMNodeStartedEvent(null, null, null));
            ContainerId completedContainerIdFromNode1 = BuilderUtils.NewContainerId(BuilderUtils
                                                                                    .NewApplicationAttemptId(BuilderUtils.NewApplicationId(0, 0), 0), 0);
            ContainerId completedContainerIdFromNode2_1 = BuilderUtils.NewContainerId(BuilderUtils
                                                                                      .NewApplicationAttemptId(BuilderUtils.NewApplicationId(1, 1), 1), 1);
            ContainerId completedContainerIdFromNode2_2 = BuilderUtils.NewContainerId(BuilderUtils
                                                                                      .NewApplicationAttemptId(BuilderUtils.NewApplicationId(1, 1), 1), 2);
            RMNodeStatusEvent statusEventFromNode1     = GetMockRMNodeStatusEvent();
            RMNodeStatusEvent statusEventFromNode2_1   = GetMockRMNodeStatusEvent();
            RMNodeStatusEvent statusEventFromNode2_2   = GetMockRMNodeStatusEvent();
            ContainerStatus   containerStatusFromNode1 = Org.Mockito.Mockito.Mock <ContainerStatus
                                                                                   >();
            ContainerStatus containerStatusFromNode2_1 = Org.Mockito.Mockito.Mock <ContainerStatus
                                                                                   >();
            ContainerStatus containerStatusFromNode2_2 = Org.Mockito.Mockito.Mock <ContainerStatus
                                                                                   >();

            Org.Mockito.Mockito.DoReturn(completedContainerIdFromNode1).When(containerStatusFromNode1
                                                                             ).GetContainerId();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatusFromNode1
                                                                           )).When(statusEventFromNode1).GetContainers();
            node.Handle(statusEventFromNode1);
            NUnit.Framework.Assert.AreEqual(1, completedContainers.Count);
            NUnit.Framework.Assert.AreEqual(completedContainerIdFromNode1, completedContainers
                                            [0].GetContainerId());
            completedContainers.Clear();
            Org.Mockito.Mockito.DoReturn(completedContainerIdFromNode2_1).When(containerStatusFromNode2_1
                                                                               ).GetContainerId();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatusFromNode2_1
                                                                           )).When(statusEventFromNode2_1).GetContainers();
            Org.Mockito.Mockito.DoReturn(completedContainerIdFromNode2_2).When(containerStatusFromNode2_2
                                                                               ).GetContainerId();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatusFromNode2_2
                                                                           )).When(statusEventFromNode2_2).GetContainers();
            node2.SetNextHeartBeat(false);
            node2.Handle(statusEventFromNode2_1);
            node2.SetNextHeartBeat(true);
            node2.Handle(statusEventFromNode2_2);
            NUnit.Framework.Assert.AreEqual(2, completedContainers.Count);
            NUnit.Framework.Assert.AreEqual(completedContainerIdFromNode2_1, completedContainers
                                            [0].GetContainerId());
            NUnit.Framework.Assert.AreEqual(completedContainerIdFromNode2_2, completedContainers
                                            [1].GetContainerId());
        }
コード例 #10
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public virtual void TestContainerLogFile()
        {
            FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
                                              ).GetAbsoluteFile();
            string        logdirwithFile = absLogDir.ToURI().ToString();
            Configuration conf           = new Configuration();

            conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
            conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
            LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();

            dirsHandler.Init(conf);
            NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
                                                                        , new ApplicationACLsManager(conf), new NMNullStateStoreService());
            // Add an application and the corresponding containers
            string        user             = "******";
            long          clusterTimeStamp = 1234;
            ApplicationId appId            = BuilderUtils.NewApplicationId(clusterTimeStamp, 1);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                >();
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);

            nmContext.GetApplications()[appId] = app;
            MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
                                                        conf, user, appId, 1);

            container.SetState(ContainerState.Running);
            nmContext.GetContainers()[containerId] = container;
            FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir
                                                        (appId.ToString(), containerId.ToString()));

            containerLogDir.Mkdirs();
            string   fileName         = "fileName";
            FilePath containerLogFile = new FilePath(containerLogDir, fileName);

            containerLogFile.CreateNewFile();
            FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user
                                                                   , nmContext);

            NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI()
                                            .ToString());
            FileUtil.FullyDelete(absLogDir);
        }
コード例 #11
0
        CreateMockedContainer(ApplicationId appId, int containerId)
        {
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId cId = BuilderUtils.NewContainerId(appAttemptId, containerId);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container c =
                Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                          >();
            Org.Mockito.Mockito.When(c.GetContainerId()).ThenReturn(cId);
            ContainerLaunchContext launchContext = Org.Mockito.Mockito.Mock <ContainerLaunchContext
                                                                             >();

            Org.Mockito.Mockito.When(c.GetLaunchContext()).ThenReturn(launchContext);
            Org.Mockito.Mockito.When(launchContext.GetApplicationACLs()).ThenReturn(new Dictionary
                                                                                    <ApplicationAccessType, string>());
            return(c);
        }
コード例 #12
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAppAttemptMetrics()
        {
            AsyncDispatcher            dispatcher = new InlineDispatcher();
            FifoScheduler              scheduler  = new FifoScheduler();
            RMApplicationHistoryWriter writer     = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                              >();
            RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
                                                    null, null, null, scheduler);

            ((RMContextImpl)rmContext).SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                                           >());
            Configuration conf = new Configuration();

            scheduler.SetRMContext(rmContext);
            scheduler.Init(conf);
            scheduler.Start();
            scheduler.Reinitialize(conf, rmContext);
            QueueMetrics         metrics             = scheduler.GetRootQueueMetrics();
            int                  beforeAppsSubmitted = metrics.GetAppsSubmitted();
            ApplicationId        appId        = BuilderUtils.NewApplicationId(200, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");

            scheduler.Handle(appEvent);
            SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false
                                                                            );

            scheduler.Handle(attemptEvent);
            appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 2);
            SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false
                                                                             );

            scheduler.Handle(attemptEvent2);
            int afterAppsSubmitted = metrics.GetAppsSubmitted();

            NUnit.Framework.Assert.AreEqual(1, afterAppsSubmitted - beforeAppsSubmitted);
            scheduler.Stop();
        }
コード例 #13
0
        public virtual void TestBlackListNodes()
        {
            Configuration conf = new Configuration();

            conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
                                                                                       ));
            MockRM rm = new MockRM(conf);

            rm.Start();
            FifoScheduler fs   = (FifoScheduler)rm.GetResourceScheduler();
            string        host = "127.0.0.1";
            RMNode        node = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, host);

            fs.Handle(new NodeAddedSchedulerEvent(node));
            ApplicationId        appId        = BuilderUtils.NewApplicationId(100, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );

            CreateMockRMApp(appAttemptId, rm.GetRMContext());
            SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "default", "user");

            fs.Handle(appEvent);
            SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false
                                                                            );

            fs.Handle(attemptEvent);
            // Verify the blacklist can be updated independent of requesting containers
            fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections
                        .EmptyList <ContainerId>(), Sharpen.Collections.SingletonList(host), null);
            NUnit.Framework.Assert.IsTrue(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted
                                              (host));
            fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections
                        .EmptyList <ContainerId>(), null, Sharpen.Collections.SingletonList(host));
            NUnit.Framework.Assert.IsFalse(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted
                                               (host));
            rm.Stop();
        }
コード例 #14
0
        public virtual void TestSortedQueues()
        {
            // Setup queue configs
            SetupSortedQueues(csConf);
            IDictionary <string, CSQueue> queues = new Dictionary <string, CSQueue>();
            CSQueue root = CapacityScheduler.ParseQueue(csContext, csConf, null, CapacitySchedulerConfiguration
                                                        .Root, queues, queues, TestUtils.spyHook);
            // Setup some nodes
            int memoryPerNode        = 10;
            int coresPerNode         = 16;
            int numNodes             = 1;
            FiCaSchedulerNode node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, memoryPerNode
                                                             * Gb);

            Org.Mockito.Mockito.DoNothing().When(node_0).ReleaseContainer(Matchers.Any <Container
                                                                                        >());
            Org.Apache.Hadoop.Yarn.Api.Records.Resource clusterResource = Resources.CreateResource
                                                                              (numNodes * (memoryPerNode * Gb), numNodes * coresPerNode);
            Org.Mockito.Mockito.When(csContext.GetNumClusterNodes()).ThenReturn(numNodes);
            // Start testing
            CSQueue a      = queues[A];
            CSQueue b      = queues[B];
            CSQueue c      = queues[C];
            CSQueue d      = queues[D];
            string  user_0 = "user_0";
            // Stub an App and its containerCompleted
            FiCaSchedulerApp app_0 = GetMockApplication(0, user_0);

            Org.Mockito.Mockito.DoReturn(true).When(app_0).ContainerCompleted(Matchers.Any <RMContainer
                                                                                            >(), Matchers.Any <ContainerStatus>(), Matchers.Any <RMContainerEventType>());
            Priority priority = TestUtils.CreateMockPriority(1);
            ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
                                                                           >();
            DrainDispatcher            drainDispatcher = new DrainDispatcher();
            RMApplicationHistoryWriter writer          = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                                   >();
            SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                         >();
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
            Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
                                                                                       );
            Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
                                                                                      ());
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(app_0.GetApplicationId
                                                                                         (), 1);
            ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);
            Container   container   = TestUtils.GetMockContainer(containerId, node_0.GetNodeID(),
                                                                 Resources.CreateResource(1 * Gb), priority);
            RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, node_0.GetNodeID
                                                              (), "user", rmContext);

            // Assign {1,2,3,4} 1GB containers respectively to queues
            StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
            StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
            StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
            StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
            root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                              ));
            for (int i = 0; i < 2; i++)
            {
                StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
                StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
                StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
                StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
                root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                                  ));
            }
            for (int i_1 = 0; i_1 < 3; i_1++)
            {
                StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
                StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
                StubQueueAllocation(c, clusterResource, node_0, 1 * Gb);
                StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
                root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                                  ));
            }
            for (int i_2 = 0; i_2 < 4; i_2++)
            {
                StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
                StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
                StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
                StubQueueAllocation(d, clusterResource, node_0, 1 * Gb);
                root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                                  ));
            }
            VerifyQueueMetrics(a, 1 * Gb, clusterResource);
            VerifyQueueMetrics(b, 2 * Gb, clusterResource);
            VerifyQueueMetrics(c, 3 * Gb, clusterResource);
            VerifyQueueMetrics(d, 4 * Gb, clusterResource);
            Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
            //Release 3 x 1GB containers from D
            for (int i_3 = 0; i_3 < 3; i_3++)
            {
                d.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
                                     .Kill, null, true);
            }
            VerifyQueueMetrics(a, 1 * Gb, clusterResource);
            VerifyQueueMetrics(b, 2 * Gb, clusterResource);
            VerifyQueueMetrics(c, 3 * Gb, clusterResource);
            VerifyQueueMetrics(d, 1 * Gb, clusterResource);
            //reset manually resources on node
            node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 1 - 2 -
                                                                      3 - 1) * Gb);
            Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
            // Assign 2 x 1GB Containers to A
            for (int i_4 = 0; i_4 < 2; i_4++)
            {
                StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
                StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
                StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
                StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
                root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                                  ));
            }
            VerifyQueueMetrics(a, 3 * Gb, clusterResource);
            VerifyQueueMetrics(b, 2 * Gb, clusterResource);
            VerifyQueueMetrics(c, 3 * Gb, clusterResource);
            VerifyQueueMetrics(d, 1 * Gb, clusterResource);
            Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
            //Release 1GB Container from A
            a.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
                                 .Kill, null, true);
            VerifyQueueMetrics(a, 2 * Gb, clusterResource);
            VerifyQueueMetrics(b, 2 * Gb, clusterResource);
            VerifyQueueMetrics(c, 3 * Gb, clusterResource);
            VerifyQueueMetrics(d, 1 * Gb, clusterResource);
            //reset manually resources on node
            node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 -
                                                                      3 - 1) * Gb);
            Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
            // Assign 1GB container to B
            StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
            StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
            StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
            StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
            root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                              ));
            VerifyQueueMetrics(a, 2 * Gb, clusterResource);
            VerifyQueueMetrics(b, 3 * Gb, clusterResource);
            VerifyQueueMetrics(c, 3 * Gb, clusterResource);
            VerifyQueueMetrics(d, 1 * Gb, clusterResource);
            Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
            //Release 1GB container resources from B
            b.CompletedContainer(clusterResource, app_0, node_0, rmContainer, null, RMContainerEventType
                                 .Kill, null, true);
            VerifyQueueMetrics(a, 2 * Gb, clusterResource);
            VerifyQueueMetrics(b, 2 * Gb, clusterResource);
            VerifyQueueMetrics(c, 3 * Gb, clusterResource);
            VerifyQueueMetrics(d, 1 * Gb, clusterResource);
            //reset manually resources on node
            node_0 = TestUtils.GetMockNode("host_0", DefaultRack, 0, (memoryPerNode - 2 - 2 -
                                                                      3 - 1) * Gb);
            Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
            // Assign 1GB container to A
            StubQueueAllocation(a, clusterResource, node_0, 1 * Gb);
            StubQueueAllocation(b, clusterResource, node_0, 0 * Gb);
            StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
            StubQueueAllocation(d, clusterResource, node_0, 0 * Gb);
            root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                              ));
            VerifyQueueMetrics(a, 3 * Gb, clusterResource);
            VerifyQueueMetrics(b, 2 * Gb, clusterResource);
            VerifyQueueMetrics(c, 3 * Gb, clusterResource);
            VerifyQueueMetrics(d, 1 * Gb, clusterResource);
            Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
            // Now do the real test, where B and D request a 1GB container
            // D should should get the next container if the order is correct
            StubQueueAllocation(a, clusterResource, node_0, 0 * Gb);
            StubQueueAllocation(b, clusterResource, node_0, 1 * Gb);
            StubQueueAllocation(c, clusterResource, node_0, 0 * Gb);
            StubQueueAllocation(d, clusterResource, node_0, 1 * Gb);
            root.AssignContainers(clusterResource, node_0, new ResourceLimits(clusterResource
                                                                              ));
            InOrder allocationOrder = Org.Mockito.Mockito.InOrder(d, b);

            allocationOrder.Verify(d).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
                                                       <FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>());
            allocationOrder.Verify(b).AssignContainers(Matchers.Eq(clusterResource), Matchers.Any
                                                       <FiCaSchedulerNode>(), Matchers.Any <ResourceLimits>());
            VerifyQueueMetrics(a, 3 * Gb, clusterResource);
            VerifyQueueMetrics(b, 2 * Gb, clusterResource);
            VerifyQueueMetrics(c, 3 * Gb, clusterResource);
            VerifyQueueMetrics(d, 2 * Gb, clusterResource);
            //D got the container
            Log.Info("status child-queues: " + ((ParentQueue)root).GetChildQueuesToPrint());
        }
コード例 #15
0
        /// <exception cref="System.Exception"/>
        public virtual void TestHeadroom()
        {
            Configuration conf = new Configuration();

            conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
                                                                                       ));
            MockRM rm = new MockRM(conf);

            rm.Start();
            FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
            // Add a node
            RMNode n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, "127.0.0.2"
                                              );

            fs.Handle(new NodeAddedSchedulerEvent(n1));
            // Add two applications
            ApplicationId        appId1        = BuilderUtils.NewApplicationId(100, 1);
            ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1,
                                                                                      1);

            CreateMockRMApp(appAttemptId1, rm.GetRMContext());
            SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");

            fs.Handle(appEvent);
            SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false
                                                                            );

            fs.Handle(attemptEvent);
            ApplicationId        appId2        = BuilderUtils.NewApplicationId(200, 2);
            ApplicationAttemptId appAttemptId2 = BuilderUtils.NewApplicationAttemptId(appId2,
                                                                                      1);

            CreateMockRMApp(appAttemptId2, rm.GetRMContext());
            SchedulerEvent appEvent2 = new AppAddedSchedulerEvent(appId2, "queue", "user");

            fs.Handle(appEvent2);
            SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false
                                                                             );

            fs.Handle(attemptEvent2);
            IList <ContainerId>     emptyId  = new AList <ContainerId>();
            IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>();
            // Set up resource requests
            // Ask for a 1 GB container for app 1
            IList <ResourceRequest> ask1 = new AList <ResourceRequest>();

            ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
                                                         .Any, BuilderUtils.NewResource(Gb, 1), 1));
            fs.Allocate(appAttemptId1, ask1, emptyId, null, null);
            // Ask for a 2 GB container for app 2
            IList <ResourceRequest> ask2 = new AList <ResourceRequest>();

            ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
                                                         .Any, BuilderUtils.NewResource(2 * Gb, 1), 1));
            fs.Allocate(appAttemptId2, ask2, emptyId, null, null);
            // Trigger container assignment
            fs.Handle(new NodeUpdateSchedulerEvent(n1));
            // Get the allocation for the applications and verify headroom
            Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation1.GetResourceLimit
                                                ().GetMemory());
            Allocation allocation2 = fs.Allocate(appAttemptId2, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation2.GetResourceLimit
                                                ().GetMemory());
            rm.Stop();
        }
コード例 #16
0
        /// <exception cref="System.Exception"/>
        public virtual void TestBlackListNodes()
        {
            Configuration conf = new Configuration();

            conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
                                                                                       ));
            MockRM rm = new MockRM(conf);

            rm.Start();
            FifoScheduler fs         = (FifoScheduler)rm.GetResourceScheduler();
            int           rack_num_0 = 0;
            int           rack_num_1 = 1;
            // Add 4 nodes in 2 racks
            // host_0_0 in rack0
            string host_0_0 = "127.0.0.1";
            RMNode n1       = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_0
                                                    );

            fs.Handle(new NodeAddedSchedulerEvent(n1));
            // host_0_1 in rack0
            string host_0_1 = "127.0.0.2";
            RMNode n2       = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_1
                                                    );

            fs.Handle(new NodeAddedSchedulerEvent(n2));
            // host_1_0 in rack1
            string host_1_0 = "127.0.0.3";
            RMNode n3       = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_0
                                                    );

            fs.Handle(new NodeAddedSchedulerEvent(n3));
            // host_1_1 in rack1
            string host_1_1 = "127.0.0.4";
            RMNode n4       = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_1
                                                    );

            fs.Handle(new NodeAddedSchedulerEvent(n4));
            // Add one application
            ApplicationId        appId1        = BuilderUtils.NewApplicationId(100, 1);
            ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1,
                                                                                      1);

            CreateMockRMApp(appAttemptId1, rm.GetRMContext());
            SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");

            fs.Handle(appEvent);
            SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false
                                                                            );

            fs.Handle(attemptEvent);
            IList <ContainerId>     emptyId  = new AList <ContainerId>();
            IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>();
            // Allow rack-locality for rack_1, but blacklist host_1_0
            // Set up resource requests
            // Ask for a 1 GB container for app 1
            IList <ResourceRequest> ask1 = new AList <ResourceRequest>();

            ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), "rack1"
                                                         , BuilderUtils.NewResource(Gb, 1), 1));
            ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
                                                         .Any, BuilderUtils.NewResource(Gb, 1), 1));
            fs.Allocate(appAttemptId1, ask1, emptyId, Sharpen.Collections.SingletonList(host_1_0
                                                                                        ), null);
            // Trigger container assignment
            fs.Handle(new NodeUpdateSchedulerEvent(n3));
            // Get the allocation for the application and verify no allocation on blacklist node
            Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation1", 0, allocation1.GetContainers().Count
                                            );
            // verify host_1_1 can get allocated as not in blacklist
            fs.Handle(new NodeUpdateSchedulerEvent(n4));
            Allocation allocation2 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation2", 1, allocation2.GetContainers().Count
                                            );
            IList <Container> containerList = allocation2.GetContainers();

            foreach (Container container in containerList)
            {
                NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container.GetNodeId
                                                    (), n4.GetNodeID());
            }
            // Ask for a 1 GB container again for app 1
            IList <ResourceRequest> ask2 = new AList <ResourceRequest>();

            // this time, rack0 is also in blacklist, so only host_1_1 is available to
            // be assigned
            ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
                                                         .Any, BuilderUtils.NewResource(Gb, 1), 1));
            fs.Allocate(appAttemptId1, ask2, emptyId, Sharpen.Collections.SingletonList("rack0"
                                                                                        ), null);
            // verify n1 is not qualified to be allocated
            fs.Handle(new NodeUpdateSchedulerEvent(n1));
            Allocation allocation3 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation3", 0, allocation3.GetContainers().Count
                                            );
            // verify n2 is not qualified to be allocated
            fs.Handle(new NodeUpdateSchedulerEvent(n2));
            Allocation allocation4 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation4", 0, allocation4.GetContainers().Count
                                            );
            // verify n3 is not qualified to be allocated
            fs.Handle(new NodeUpdateSchedulerEvent(n3));
            Allocation allocation5 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation5", 0, allocation5.GetContainers().Count
                                            );
            fs.Handle(new NodeUpdateSchedulerEvent(n4));
            Allocation allocation6 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation6", 1, allocation6.GetContainers().Count
                                            );
            containerList = allocation6.GetContainers();
            foreach (Container container_1 in containerList)
            {
                NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container_1.GetNodeId
                                                    (), n4.GetNodeID());
            }
            rm.Stop();
        }
コード例 #17
0
        public virtual void TestReleaseWhileRunning()
        {
            DrainDispatcher drainDispatcher = new DrainDispatcher();
            EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
                                                                      <EventHandler>();
            EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();

            drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
            drainDispatcher.Register(typeof(RMNodeEventType), generic);
            drainDispatcher.Init(new YarnConfiguration());
            drainDispatcher.Start();
            NodeId               nodeId       = BuilderUtils.NewNodeId("host", 3425);
            ApplicationId        appId        = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId            = BuilderUtils.NewContainerId(appAttemptId, 1);
            ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
                                                                           >();
            Resource  resource  = BuilderUtils.NewResource(512, 1);
            Priority  priority  = BuilderUtils.NewPriority(5);
            Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
                                                            resource, priority, null);
            ConcurrentMap <ApplicationId, RMApp> rmApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
                                                                                  <ApplicationId, RMApp>());
            RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
                                                               ())).ThenReturn(null);
            Org.Mockito.Mockito.DoReturn(rmApp).When(rmApps)[(ApplicationId)Matchers.Any()];
            RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                          >();
            SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                         >();
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
            Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApps()).ThenReturn(rmApps);
            Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
                                                                                       );
            Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
                                                                                      ());
            RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
                                                          , rmContext);

            NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
            NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
            NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
            Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
                                                                       ), Matchers.AnyLong());
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
                                                ));
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
                                            );
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
                                            , rmContainer.GetLogURL());
            // In RUNNING state. Verify RELEASED and associated actions.
            Org.Mockito.Mockito.Reset(appAttemptEventHandler);
            ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
                                                                                           , SchedulerUtils.ReleasedContainer);

            rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
                                                            .Released));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
                                            );
            NUnit.Framework.Assert.AreEqual(SchedulerUtils.ReleasedContainer, rmContainer.GetDiagnosticsInfo
                                                ());
            NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, rmContainer.GetContainerExitStatus
                                                ());
            NUnit.Framework.Assert.AreEqual(ContainerState.Complete, rmContainer.GetContainerState
                                                ());
            Org.Mockito.Mockito.Verify(writer).ContainerFinished(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerFinished(Matchers.Any <RMContainer>
                                                                        (), Matchers.AnyLong());
            ArgumentCaptor <RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor.ForClass
                                                                         <RMAppAttemptContainerFinishedEvent>();

            Org.Mockito.Mockito.Verify(appAttemptEventHandler).Handle(captor.Capture());
            RMAppAttemptContainerFinishedEvent cfEvent = captor.GetValue();

            NUnit.Framework.Assert.AreEqual(appAttemptId, cfEvent.GetApplicationAttemptId());
            NUnit.Framework.Assert.AreEqual(containerStatus, cfEvent.GetContainerStatus());
            NUnit.Framework.Assert.AreEqual(RMAppAttemptEventType.ContainerFinished, cfEvent.
                                            GetType());
            // In RELEASED state. A FINIHSED event may come in.
            rmContainer.Handle(new RMContainerFinishedEvent(containerId, SchedulerUtils.CreateAbnormalContainerStatus
                                                                (containerId, "FinishedContainer"), RMContainerEventType.Finished));
            NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
                                            );
        }
コード例 #18
0
        public virtual void TestExpireWhileRunning()
        {
            DrainDispatcher drainDispatcher = new DrainDispatcher();
            EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
                                                                      <EventHandler>();
            EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();

            drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
            drainDispatcher.Register(typeof(RMNodeEventType), generic);
            drainDispatcher.Init(new YarnConfiguration());
            drainDispatcher.Start();
            NodeId               nodeId       = BuilderUtils.NewNodeId("host", 3425);
            ApplicationId        appId        = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId            = BuilderUtils.NewContainerId(appAttemptId, 1);
            ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
                                                                           >();
            Resource  resource  = BuilderUtils.NewResource(512, 1);
            Priority  priority  = BuilderUtils.NewPriority(5);
            Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
                                                            resource, priority, null);
            RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                          >();
            SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                         >();
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
            Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
                                                                                       );
            Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
                                                                                      ());
            RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
                                                          , rmContext);

            NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
            NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
            NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
            Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
                                                                       ), Matchers.AnyLong());
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
                                                ));
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
                                            );
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
                                            , rmContainer.GetLogURL());
            // In RUNNING state. Verify EXPIRE and associated actions.
            Org.Mockito.Mockito.Reset(appAttemptEventHandler);
            ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
                                                                                           , SchedulerUtils.ExpiredContainer);

            rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
                                                            .Expire));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            Org.Mockito.Mockito.Verify(writer, Org.Mockito.Mockito.Never()).ContainerFinished
                (Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Never()).ContainerFinished
                (Matchers.Any <RMContainer>(), Matchers.AnyLong());
        }
コード例 #19
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestContainerLogPageAccess()
        {
            // SecureIOUtils require Native IO to be enabled. This test will run
            // only if it is enabled.
            Assume.AssumeTrue(NativeIO.IsAvailable());
            string   user         = "******" + Runtime.CurrentTimeMillis();
            FilePath absLogDir    = null;
            FilePath appDir       = null;
            FilePath containerDir = null;
            FilePath syslog       = null;

            try
            {
                // target log directory
                absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir")
                            .GetAbsoluteFile();
                absLogDir.Mkdir();
                Configuration conf = new Configuration();
                conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString());
                conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
                UserGroupInformation.SetConfiguration(conf);
                NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
                healthChecker.Init(conf);
                LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
                // Add an application and the corresponding containers
                RecordFactory recordFactory    = RecordFactoryProvider.GetRecordFactory(conf);
                long          clusterTimeStamp = 1234;
                ApplicationId appId            = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
                                                                               , 1);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                    app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                    >();
                Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
                // Making sure that application returns a random user. This is required
                // for SecureIOUtils' file owner check.
                Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
                ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                         );
                ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                     , 0);
                // Testing secure read access for log files
                // Creating application and container directory and syslog file.
                appDir = new FilePath(absLogDir, appId.ToString());
                appDir.Mkdir();
                containerDir = new FilePath(appDir, container1.ToString());
                containerDir.Mkdir();
                syslog = new FilePath(containerDir, "syslog");
                syslog.CreateNewFile();
                BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog)
                                                                     );
                @out.Write(Sharpen.Runtime.GetBytesForString("Log file Content"));
                @out.Close();
                Context context = Org.Mockito.Mockito.Mock <Context>();
                ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                               > appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                                 >();
                appMap[appId] = app;
                Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap);
                ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                   > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                                                         >();
                Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
                Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler);
                MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
                                                            conf, user, appId, 1);
                container.SetState(ContainerState.Running);
                context.GetContainers()[container1] = container;
                ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock
                                                                       (context);
                IDictionary <string, string> @params = new Dictionary <string, string>();
                @params[YarnWebParams.ContainerId]      = container1.ToString();
                @params[YarnWebParams.ContainerLogType] = "syslog";
                Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof(
                                                                                                     ContainerLogsPage), cLogsBlock, @params, (Module[])null);
                PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
                Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '"
                                                        + user + "' doesn't own requested log file : syslog");
            }
            finally
            {
                if (syslog != null)
                {
                    syslog.Delete();
                }
                if (containerDir != null)
                {
                    containerDir.Delete();
                }
                if (appDir != null)
                {
                    appDir.Delete();
                }
                if (absLogDir != null)
                {
                    absLogDir.Delete();
                }
            }
        }
コード例 #20
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public virtual void TestContainerLogDirs()
        {
            FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
                                              ).GetAbsoluteFile();
            string        logdirwithFile = absLogDir.ToURI().ToString();
            Configuration conf           = new Configuration();

            conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
            NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();

            healthChecker.Init(conf);
            LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();

            NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
                                                                        , new ApplicationACLsManager(conf), new NMNullStateStoreService());
            // Add an application and the corresponding containers
            RecordFactory recordFactory    = RecordFactoryProvider.GetRecordFactory(conf);
            string        user             = "******";
            long          clusterTimeStamp = 1234;
            ApplicationId appId            = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
                                                                           , 1);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                >();
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                 , 0);

            nmContext.GetApplications()[appId] = app;
            MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
                                                        conf, user, appId, 1);

            container.SetState(ContainerState.Running);
            nmContext.GetContainers()[container1] = container;
            IList <FilePath> files = null;

            files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
            NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
            // After container is completed, it is removed from nmContext
            Sharpen.Collections.Remove(nmContext.GetContainers(), container1);
            NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]);
            files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
            NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
            // Create a new context to check if correct container log dirs are fetched
            // on full disk.
            LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler
                                                                                     );

            // good log dirs are empty and nm log dir is in the full log dir list.
            Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList
                                                                                     <string>());
            Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays
                                                                                            .AsList(new string[] { absLogDir.GetAbsolutePath() }));
            nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager
                                                      (conf), new NMNullStateStoreService());
            nmContext.GetApplications()[appId] = app;
            container.SetState(ContainerState.Running);
            nmContext.GetContainers()[container1] = container;
            IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext
                                                                           );
            FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1);

            NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir));
        }