Esempio n. 1
0
        public virtual void TestUpdateHeartbeatResponseForCleanup()
        {
            RMNodeImpl node   = GetRunningNode();
            NodeId     nodeId = node.GetNodeID();
            // Expire a container
            ContainerId completedContainerId = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId
                                                                               (BuilderUtils.NewApplicationId(0, 0), 0), 0);

            node.Handle(new RMNodeCleanContainerEvent(nodeId, completedContainerId));
            NUnit.Framework.Assert.AreEqual(1, node.GetContainersToCleanUp().Count);
            // Finish an application
            ApplicationId finishedAppId = BuilderUtils.NewApplicationId(0, 1);

            node.Handle(new RMNodeCleanAppEvent(nodeId, finishedAppId));
            NUnit.Framework.Assert.AreEqual(1, node.GetAppsToCleanup().Count);
            // Verify status update does not clear containers/apps to cleanup
            // but updating heartbeat response for cleanup does
            RMNodeStatusEvent statusEvent = GetMockRMNodeStatusEvent();

            node.Handle(statusEvent);
            NUnit.Framework.Assert.AreEqual(1, node.GetContainersToCleanUp().Count);
            NUnit.Framework.Assert.AreEqual(1, node.GetAppsToCleanup().Count);
            NodeHeartbeatResponse hbrsp = Org.Apache.Hadoop.Yarn.Util.Records.NewRecord <NodeHeartbeatResponse
                                                                                         >();

            node.UpdateNodeHeartbeatResponseForCleanup(hbrsp);
            NUnit.Framework.Assert.AreEqual(0, node.GetContainersToCleanUp().Count);
            NUnit.Framework.Assert.AreEqual(0, node.GetAppsToCleanup().Count);
            NUnit.Framework.Assert.AreEqual(1, hbrsp.GetContainersToCleanup().Count);
            NUnit.Framework.Assert.AreEqual(completedContainerId, hbrsp.GetContainersToCleanup
                                                ()[0]);
            NUnit.Framework.Assert.AreEqual(1, hbrsp.GetApplicationsToCleanup().Count);
            NUnit.Framework.Assert.AreEqual(finishedAppId, hbrsp.GetApplicationsToCleanup()[0
                                            ]);
        }
Esempio n. 2
0
 public virtual void TestGetPathApplicationId()
 {
     NUnit.Framework.Assert.AreEqual("/proxy/application_100_0001", ProxyUriUtils.GetPath
                                         (BuilderUtils.NewApplicationId(100l, 1)));
     NUnit.Framework.Assert.AreEqual("/proxy/application_6384623_0005", ProxyUriUtils.
                                     GetPath(BuilderUtils.NewApplicationId(6384623l, 5)));
 }
Esempio n. 3
0
        public virtual void TestGetProxyUriFromPluginsReturnsNullIfNoPlugins()
        {
            ApplicationId             id   = BuilderUtils.NewApplicationId(6384623l, 5);
            IList <TrackingUriPlugin> list = Lists.NewArrayListWithExpectedSize(0);

            NUnit.Framework.Assert.IsNull(ProxyUriUtils.GetUriFromTrackingPlugins(id, list));
        }
Esempio n. 4
0
        public virtual void TestStatusChange()
        {
            //Start the node
            node.Handle(new RMNodeStartedEvent(null, null, null));
            //Add info to the queue first
            node.SetNextHeartBeat(false);
            ContainerId completedContainerId1 = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId
                                                                                (BuilderUtils.NewApplicationId(0, 0), 0), 0);
            ContainerId completedContainerId2 = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId
                                                                                (BuilderUtils.NewApplicationId(1, 1), 1), 1);
            RMNodeStatusEvent statusEvent1     = GetMockRMNodeStatusEvent();
            RMNodeStatusEvent statusEvent2     = GetMockRMNodeStatusEvent();
            ContainerStatus   containerStatus1 = Org.Mockito.Mockito.Mock <ContainerStatus>();
            ContainerStatus   containerStatus2 = Org.Mockito.Mockito.Mock <ContainerStatus>();

            Org.Mockito.Mockito.DoReturn(completedContainerId1).When(containerStatus1).GetContainerId
                ();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatus1))
            .When(statusEvent1).GetContainers();
            Org.Mockito.Mockito.DoReturn(completedContainerId2).When(containerStatus2).GetContainerId
                ();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatus2))
            .When(statusEvent2).GetContainers();
            Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(1)).Handle(Matchers.Any
                                                                                       <NodeUpdateSchedulerEvent>());
            node.Handle(statusEvent1);
            node.Handle(statusEvent2);
            Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(1)).Handle(Matchers.Any
                                                                                       <NodeUpdateSchedulerEvent>());
            NUnit.Framework.Assert.AreEqual(2, node.GetQueueSize());
            node.Handle(new RMNodeEvent(node.GetNodeID(), RMNodeEventType.Expire));
            NUnit.Framework.Assert.AreEqual(0, node.GetQueueSize());
        }
Esempio n. 5
0
        public virtual void TestExpiredContainer()
        {
            // Start the node
            node.Handle(new RMNodeStartedEvent(null, null, null));
            Org.Mockito.Mockito.Verify(scheduler).Handle(Matchers.Any <NodeAddedSchedulerEvent
                                                                       >());
            // Expire a container
            ContainerId completedContainerId = BuilderUtils.NewContainerId(BuilderUtils.NewApplicationAttemptId
                                                                               (BuilderUtils.NewApplicationId(0, 0), 0), 0);

            node.Handle(new RMNodeCleanContainerEvent(null, completedContainerId));
            NUnit.Framework.Assert.AreEqual(1, node.GetContainersToCleanUp().Count);
            // Now verify that scheduler isn't notified of an expired container
            // by checking number of 'completedContainers' it got in the previous event
            RMNodeStatusEvent statusEvent     = GetMockRMNodeStatusEvent();
            ContainerStatus   containerStatus = Org.Mockito.Mockito.Mock <ContainerStatus>();

            Org.Mockito.Mockito.DoReturn(completedContainerId).When(containerStatus).GetContainerId
                ();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatus)).
            When(statusEvent).GetContainers();
            node.Handle(statusEvent);

            /* Expect the scheduler call handle function 2 times
             * 1. RMNode status from new to Running, handle the add_node event
             * 2. handle the node update event
             */
            Org.Mockito.Mockito.Verify(scheduler, Org.Mockito.Mockito.Times(2)).Handle(Matchers.Any
                                                                                       <NodeUpdateSchedulerEvent>());
        }
Esempio n. 6
0
        public virtual void TestNMWebApp()
        {
            Context       nmContext    = new NodeManager.NMContext(null, null, null, null, null);
            ResourceView  resourceView = new _ResourceView_147();
            Configuration conf         = new Configuration();

            conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
            conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
            NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();

            healthChecker.Init(conf);
            LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
            WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
                                                 (conf), dirsHandler);

            server.Init(conf);
            server.Start();
            // Add an application and the corresponding containers
            RecordFactory recordFactory    = RecordFactoryProvider.GetRecordFactory(conf);
            Dispatcher    dispatcher       = new AsyncDispatcher();
            string        user             = "******";
            long          clusterTimeStamp = 1234;
            ApplicationId appId            = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
                                                                           , 1);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                >();
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
            nmContext.GetApplications()[appId] = app;
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                 , 0);
            ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                 , 1);
            NodeManagerMetrics  metrics    = Org.Mockito.Mockito.Mock <NodeManagerMetrics>();
            NMStateStoreService stateStore = new NMNullStateStoreService();

            foreach (ContainerId containerId in new ContainerId[] { container1, container2 })
            {
                // TODO: Use builder utils
                ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                        >();
                long  currentTime    = Runtime.CurrentTimeMillis();
                Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234
                                                                      , user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString
                                                                          ("password"), currentTime);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
                    = new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics
                                             , BuilderUtils.NewContainerTokenIdentifier(containerToken));
                nmContext.GetContainers()[containerId] = container;
                //TODO: Gross hack. Fix in code.
                ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
                                                  ();
                nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container;
                WriteContainerLogs(nmContext, containerId, dirsHandler);
            }
        }
Esempio n. 7
0
            internal WrappedApplication(TestApplication _enclosing, int id, long timestamp, string
                                        user, int numContainers)
            {
                this._enclosing = _enclosing;
                Configuration conf = new Configuration();

                this.dispatcher = new DrainDispatcher();
                this.containerTokenIdentifierMap = new Dictionary <ContainerId, ContainerTokenIdentifier
                                                                   >();
                this.dispatcher.Init(conf);
                this.localizerBus      = Org.Mockito.Mockito.Mock <EventHandler>();
                this.launcherBus       = Org.Mockito.Mockito.Mock <EventHandler>();
                this.monitorBus        = Org.Mockito.Mockito.Mock <EventHandler>();
                this.auxBus            = Org.Mockito.Mockito.Mock <EventHandler>();
                this.containerBus      = Org.Mockito.Mockito.Mock <EventHandler>();
                this.logAggregationBus = Org.Mockito.Mockito.Mock <EventHandler>();
                this.dispatcher.Register(typeof(LocalizationEventType), this.localizerBus);
                this.dispatcher.Register(typeof(ContainersLauncherEventType), this.launcherBus);
                this.dispatcher.Register(typeof(ContainersMonitorEventType), this.monitorBus);
                this.dispatcher.Register(typeof(AuxServicesEventType), this.auxBus);
                this.dispatcher.Register(typeof(ContainerEventType), this.containerBus);
                this.dispatcher.Register(typeof(LogHandlerEventType), this.logAggregationBus);
                this.nmTokenSecretMgr = Org.Mockito.Mockito.Mock <NMTokenSecretManagerInNM>();
                this.context          = Org.Mockito.Mockito.Mock <Context>();
                Org.Mockito.Mockito.When(this.context.GetContainerTokenSecretManager()).ThenReturn
                    (new NMContainerTokenSecretManager(conf));
                Org.Mockito.Mockito.When(this.context.GetApplicationACLsManager()).ThenReturn(new
                                                                                              ApplicationACLsManager(conf));
                Org.Mockito.Mockito.When(this.context.GetNMTokenSecretManager()).ThenReturn(this.
                                                                                            nmTokenSecretMgr);
                // Setting master key
                MasterKey masterKey = new MasterKeyPBImpl();

                masterKey.SetKeyId(123);
                masterKey.SetBytes(ByteBuffer.Wrap(new byte[] { (123) }));
                this.context.GetContainerTokenSecretManager().SetMasterKey(masterKey);
                this.user  = user;
                this.appId = BuilderUtils.NewApplicationId(timestamp, id);
                this.app   = new ApplicationImpl(this.dispatcher, this.user, this.appId, null, this
                                                 .context);
                this.containers = new AList <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                             >();
                for (int i = 0; i < numContainers; i++)
                {
                    Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
                        = this._enclosing.CreateMockedContainer(this.appId, i);
                    this.containers.AddItem(container);
                    long currentTime = Runtime.CurrentTimeMillis();
                    ContainerTokenIdentifier identifier = new ContainerTokenIdentifier(container.GetContainerId
                                                                                           (), string.Empty, string.Empty, null, currentTime + 2000, masterKey.GetKeyId(),
                                                                                       currentTime, Priority.NewInstance(0), 0);
                    this.containerTokenIdentifierMap[identifier.GetContainerID()] = identifier;
                    this.context.GetContainerTokenSecretManager().StartContainerSuccessful(identifier
                                                                                           );
                    NUnit.Framework.Assert.IsFalse(this.context.GetContainerTokenSecretManager().IsValidStartContainerRequest
                                                       (identifier));
                }
                this.dispatcher.Start();
            }
Esempio n. 8
0
 public virtual void TestGetPathAndQuery()
 {
     NUnit.Framework.Assert.AreEqual("/proxy/application_6384623_0005/static/app?foo=bar"
                                     , ProxyUriUtils.GetPathAndQuery(BuilderUtils.NewApplicationId(6384623l, 5), "/static/app"
                                                                     , "?foo=bar", false));
     NUnit.Framework.Assert.AreEqual("/proxy/application_6384623_0005/static/app?foo=bar&bad=good&proxyapproved=true"
                                     , ProxyUriUtils.GetPathAndQuery(BuilderUtils.NewApplicationId(6384623l, 5), "/static/app"
                                                                     , "foo=bar&bad=good", true));
 }
Esempio n. 9
0
        public virtual void TestGetProxyUriNull()
        {
            URI           originalUri = null;
            URI           proxyUri    = new URI("http://proxy.net:8080/");
            ApplicationId id          = BuilderUtils.NewApplicationId(6384623l, 5);
            URI           expected    = new URI("http://proxy.net:8080/proxy/application_6384623_0005/");
            URI           result      = ProxyUriUtils.GetProxyUri(originalUri, proxyUri, id);

            NUnit.Framework.Assert.AreEqual(expected, result);
        }
Esempio n. 10
0
 public virtual void TestGetPathApplicationIdString()
 {
     NUnit.Framework.Assert.AreEqual("/proxy/application_6384623_0005", ProxyUriUtils.
                                     GetPath(BuilderUtils.NewApplicationId(6384623l, 5), null));
     NUnit.Framework.Assert.AreEqual("/proxy/application_6384623_0005/static/app", ProxyUriUtils
                                     .GetPath(BuilderUtils.NewApplicationId(6384623l, 5), "/static/app"));
     NUnit.Framework.Assert.AreEqual("/proxy/application_6384623_0005/", ProxyUriUtils
                                     .GetPath(BuilderUtils.NewApplicationId(6384623l, 5), "/"));
     NUnit.Framework.Assert.AreEqual("/proxy/application_6384623_0005/some/path", ProxyUriUtils
                                     .GetPath(BuilderUtils.NewApplicationId(6384623l, 5), "some/path"));
 }
Esempio n. 11
0
        private static AppSchedulingInfo MockApp(string user)
        {
            AppSchedulingInfo app = Org.Mockito.Mockito.Mock <AppSchedulingInfo>();

            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            ApplicationId        appId = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId id    = BuilderUtils.NewApplicationAttemptId(appId, 1);

            Org.Mockito.Mockito.When(app.GetApplicationAttemptId()).ThenReturn(id);
            return(app);
        }
Esempio n. 12
0
        public MockApp(string user, long clusterTimeStamp, int uniqId)
            : base()
        {
            this.user = user;
            // Add an application and the corresponding containers
            RecordFactory recordFactory = RecordFactoryProvider.GetRecordFactory(new Configuration
                                                                                     ());

            this.appId = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp, uniqId
                                                       );
            appState = ApplicationState.New;
        }
 public virtual void Setup()
 {
     mockDelService  = Org.Mockito.Mockito.Mock <DeletionService>();
     conf            = new YarnConfiguration();
     dispatcher      = CreateDispatcher(conf);
     appEventHandler = Org.Mockito.Mockito.Mock <EventHandler>();
     dispatcher.Register(typeof(ApplicationEventType), appEventHandler);
     appId        = BuilderUtils.NewApplicationId(1234, 1);
     appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1);
     container11  = BuilderUtils.NewContainerId(appAttemptId, 1);
     dirsHandler  = new LocalDirsHandlerService();
 }
Esempio n. 14
0
        public static ApplicationAttemptId GetMockApplicationAttemptId(int appId, int attemptId
                                                                       )
        {
            ApplicationId        applicationId        = BuilderUtils.NewApplicationId(0l, appId);
            ApplicationAttemptId applicationAttemptId = Org.Mockito.Mockito.Mock <ApplicationAttemptId
                                                                                  >();

            Org.Mockito.Mockito.When(applicationAttemptId.GetApplicationId()).ThenReturn(applicationId
                                                                                         );
            Org.Mockito.Mockito.When(applicationAttemptId.GetAttemptId()).ThenReturn(attemptId
                                                                                     );
            return(applicationAttemptId);
        }
Esempio n. 15
0
        public virtual void TestGetProxyUriFromPluginsReturnsValidUriWhenAble()
        {
            ApplicationId             id   = BuilderUtils.NewApplicationId(6384623l, 5);
            IList <TrackingUriPlugin> list = Lists.NewArrayListWithExpectedSize(2);

            // Insert a plugin that returns null.
            list.AddItem(new _TrackingUriPlugin_108());
            // Insert a plugin that returns a valid URI.
            list.AddItem(new _TrackingUriPlugin_114());
            URI result = ProxyUriUtils.GetUriFromTrackingPlugins(id, list);

            NUnit.Framework.Assert.IsNotNull(result);
        }
Esempio n. 16
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public virtual void TestContainerLogFile()
        {
            FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
                                              ).GetAbsoluteFile();
            string        logdirwithFile = absLogDir.ToURI().ToString();
            Configuration conf           = new Configuration();

            conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
            conf.SetFloat(YarnConfiguration.NmMaxPerDiskUtilizationPercentage, 0.0f);
            LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();

            dirsHandler.Init(conf);
            NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
                                                                        , new ApplicationACLsManager(conf), new NMNullStateStoreService());
            // Add an application and the corresponding containers
            string        user             = "******";
            long          clusterTimeStamp = 1234;
            ApplicationId appId            = BuilderUtils.NewApplicationId(clusterTimeStamp, 1);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                >();
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId = BuilderUtils.NewContainerId(appAttemptId, 1);

            nmContext.GetApplications()[appId] = app;
            MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
                                                        conf, user, appId, 1);

            container.SetState(ContainerState.Running);
            nmContext.GetContainers()[containerId] = container;
            FilePath containerLogDir = new FilePath(absLogDir, ContainerLaunch.GetRelativeContainerLogDir
                                                        (appId.ToString(), containerId.ToString()));

            containerLogDir.Mkdirs();
            string   fileName         = "fileName";
            FilePath containerLogFile = new FilePath(containerLogDir, fileName);

            containerLogFile.CreateNewFile();
            FilePath file = ContainerLogsUtils.GetContainerLogFile(containerId, fileName, user
                                                                   , nmContext);

            NUnit.Framework.Assert.AreEqual(containerLogFile.ToURI().ToString(), file.ToURI()
                                            .ToString());
            FileUtil.FullyDelete(absLogDir);
        }
Esempio n. 17
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAppAttemptMetrics()
        {
            AsyncDispatcher            dispatcher = new InlineDispatcher();
            FifoScheduler              scheduler  = new FifoScheduler();
            RMApplicationHistoryWriter writer     = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                              >();
            RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null, null,
                                                    null, null, null, scheduler);

            ((RMContextImpl)rmContext).SetSystemMetricsPublisher(Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                                           >());
            Configuration conf = new Configuration();

            scheduler.SetRMContext(rmContext);
            scheduler.Init(conf);
            scheduler.Start();
            scheduler.Reinitialize(conf, rmContext);
            QueueMetrics         metrics             = scheduler.GetRootQueueMetrics();
            int                  beforeAppsSubmitted = metrics.GetAppsSubmitted();
            ApplicationId        appId        = BuilderUtils.NewApplicationId(200, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "queue", "user");

            scheduler.Handle(appEvent);
            SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false
                                                                            );

            scheduler.Handle(attemptEvent);
            appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 2);
            SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId, false
                                                                             );

            scheduler.Handle(attemptEvent2);
            int afterAppsSubmitted = metrics.GetAppsSubmitted();

            NUnit.Framework.Assert.AreEqual(1, afterAppsSubmitted - beforeAppsSubmitted);
            scheduler.Stop();
        }
Esempio n. 18
0
        public virtual void TestBlackListNodes()
        {
            Configuration conf = new Configuration();

            conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
                                                                                       ));
            MockRM rm = new MockRM(conf);

            rm.Start();
            FifoScheduler fs   = (FifoScheduler)rm.GetResourceScheduler();
            string        host = "127.0.0.1";
            RMNode        node = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, host);

            fs.Handle(new NodeAddedSchedulerEvent(node));
            ApplicationId        appId        = BuilderUtils.NewApplicationId(100, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );

            CreateMockRMApp(appAttemptId, rm.GetRMContext());
            SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId, "default", "user");

            fs.Handle(appEvent);
            SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false
                                                                            );

            fs.Handle(attemptEvent);
            // Verify the blacklist can be updated independent of requesting containers
            fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections
                        .EmptyList <ContainerId>(), Sharpen.Collections.SingletonList(host), null);
            NUnit.Framework.Assert.IsTrue(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted
                                              (host));
            fs.Allocate(appAttemptId, Sharpen.Collections.EmptyList <ResourceRequest>(), Sharpen.Collections
                        .EmptyList <ContainerId>(), null, Sharpen.Collections.SingletonList(host));
            NUnit.Framework.Assert.IsFalse(fs.GetApplicationAttempt(appAttemptId).IsBlacklisted
                                               (host));
            rm.Stop();
        }
Esempio n. 19
0
        public virtual void TestReleaseWhileRunning()
        {
            DrainDispatcher drainDispatcher = new DrainDispatcher();
            EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
                                                                      <EventHandler>();
            EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();

            drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
            drainDispatcher.Register(typeof(RMNodeEventType), generic);
            drainDispatcher.Init(new YarnConfiguration());
            drainDispatcher.Start();
            NodeId               nodeId       = BuilderUtils.NewNodeId("host", 3425);
            ApplicationId        appId        = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId            = BuilderUtils.NewContainerId(appAttemptId, 1);
            ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
                                                                           >();
            Resource  resource  = BuilderUtils.NewResource(512, 1);
            Priority  priority  = BuilderUtils.NewPriority(5);
            Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
                                                            resource, priority, null);
            ConcurrentMap <ApplicationId, RMApp> rmApps = Org.Mockito.Mockito.Spy(new ConcurrentHashMap
                                                                                  <ApplicationId, RMApp>());
            RMApp rmApp = Org.Mockito.Mockito.Mock <RMApp>();

            Org.Mockito.Mockito.When(rmApp.GetRMAppAttempt((ApplicationAttemptId)Matchers.Any
                                                               ())).ThenReturn(null);
            Org.Mockito.Mockito.DoReturn(rmApp).When(rmApps)[(ApplicationId)Matchers.Any()];
            RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                          >();
            SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                         >();
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
            Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApps()).ThenReturn(rmApps);
            Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
                                                                                       );
            Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
                                                                                      ());
            RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
                                                          , rmContext);

            NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
            NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
            NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
            Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
                                                                       ), Matchers.AnyLong());
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
                                                ));
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
                                            );
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
                                            , rmContainer.GetLogURL());
            // In RUNNING state. Verify RELEASED and associated actions.
            Org.Mockito.Mockito.Reset(appAttemptEventHandler);
            ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
                                                                                           , SchedulerUtils.ReleasedContainer);

            rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
                                                            .Released));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
                                            );
            NUnit.Framework.Assert.AreEqual(SchedulerUtils.ReleasedContainer, rmContainer.GetDiagnosticsInfo
                                                ());
            NUnit.Framework.Assert.AreEqual(ContainerExitStatus.Aborted, rmContainer.GetContainerExitStatus
                                                ());
            NUnit.Framework.Assert.AreEqual(ContainerState.Complete, rmContainer.GetContainerState
                                                ());
            Org.Mockito.Mockito.Verify(writer).ContainerFinished(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerFinished(Matchers.Any <RMContainer>
                                                                        (), Matchers.AnyLong());
            ArgumentCaptor <RMAppAttemptContainerFinishedEvent> captor = ArgumentCaptor.ForClass
                                                                         <RMAppAttemptContainerFinishedEvent>();

            Org.Mockito.Mockito.Verify(appAttemptEventHandler).Handle(captor.Capture());
            RMAppAttemptContainerFinishedEvent cfEvent = captor.GetValue();

            NUnit.Framework.Assert.AreEqual(appAttemptId, cfEvent.GetApplicationAttemptId());
            NUnit.Framework.Assert.AreEqual(containerStatus, cfEvent.GetContainerStatus());
            NUnit.Framework.Assert.AreEqual(RMAppAttemptEventType.ContainerFinished, cfEvent.
                                            GetType());
            // In RELEASED state. A FINIHSED event may come in.
            rmContainer.Handle(new RMContainerFinishedEvent(containerId, SchedulerUtils.CreateAbnormalContainerStatus
                                                                (containerId, "FinishedContainer"), RMContainerEventType.Finished));
            NUnit.Framework.Assert.AreEqual(RMContainerState.Released, rmContainer.GetState()
                                            );
        }
Esempio n. 20
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestContainerLogPageAccess()
        {
            // SecureIOUtils require Native IO to be enabled. This test will run
            // only if it is enabled.
            Assume.AssumeTrue(NativeIO.IsAvailable());
            string   user         = "******" + Runtime.CurrentTimeMillis();
            FilePath absLogDir    = null;
            FilePath appDir       = null;
            FilePath containerDir = null;
            FilePath syslog       = null;

            try
            {
                // target log directory
                absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir")
                            .GetAbsoluteFile();
                absLogDir.Mkdir();
                Configuration conf = new Configuration();
                conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString());
                conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
                UserGroupInformation.SetConfiguration(conf);
                NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
                healthChecker.Init(conf);
                LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
                // Add an application and the corresponding containers
                RecordFactory recordFactory    = RecordFactoryProvider.GetRecordFactory(conf);
                long          clusterTimeStamp = 1234;
                ApplicationId appId            = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
                                                                               , 1);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                    app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                    >();
                Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
                // Making sure that application returns a random user. This is required
                // for SecureIOUtils' file owner check.
                Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
                ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                         );
                ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                     , 0);
                // Testing secure read access for log files
                // Creating application and container directory and syslog file.
                appDir = new FilePath(absLogDir, appId.ToString());
                appDir.Mkdir();
                containerDir = new FilePath(appDir, container1.ToString());
                containerDir.Mkdir();
                syslog = new FilePath(containerDir, "syslog");
                syslog.CreateNewFile();
                BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog)
                                                                     );
                @out.Write(Sharpen.Runtime.GetBytesForString("Log file Content"));
                @out.Close();
                Context context = Org.Mockito.Mockito.Mock <Context>();
                ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                               > appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                                 >();
                appMap[appId] = app;
                Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap);
                ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                   > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                                                         >();
                Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
                Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler);
                MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
                                                            conf, user, appId, 1);
                container.SetState(ContainerState.Running);
                context.GetContainers()[container1] = container;
                ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock
                                                                       (context);
                IDictionary <string, string> @params = new Dictionary <string, string>();
                @params[YarnWebParams.ContainerId]      = container1.ToString();
                @params[YarnWebParams.ContainerLogType] = "syslog";
                Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof(
                                                                                                     ContainerLogsPage), cLogsBlock, @params, (Module[])null);
                PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
                Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '"
                                                        + user + "' doesn't own requested log file : syslog");
            }
            finally
            {
                if (syslog != null)
                {
                    syslog.Delete();
                }
                if (containerDir != null)
                {
                    containerDir.Delete();
                }
                if (appDir != null)
                {
                    appDir.Delete();
                }
                if (absLogDir != null)
                {
                    absLogDir.Delete();
                }
            }
        }
Esempio n. 21
0
        /// <exception cref="System.IO.IOException"/>
        private void VerifyTokenWithTamperedID(Configuration conf, TestClientToAMTokens.CustomAM
                                               am, Org.Apache.Hadoop.Security.Token.Token <ClientToAMTokenIdentifier> token)
        {
            // Malicious user, messes with appId
            UserGroupInformation      ugi         = UserGroupInformation.CreateRemoteUser("me");
            ClientToAMTokenIdentifier maliciousID = new ClientToAMTokenIdentifier(BuilderUtils
                                                                                  .NewApplicationAttemptId(BuilderUtils.NewApplicationId(am.appAttemptId.GetApplicationId
                                                                                                                                             ().GetClusterTimestamp(), 42), 43), UserGroupInformation.GetCurrentUser().GetShortUserName
                                                                                      ());

            VerifyTamperedToken(conf, am, token, ugi, maliciousID);
        }
Esempio n. 22
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public virtual void TestContainerLogDirs()
        {
            FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
                                              ).GetAbsoluteFile();
            string        logdirwithFile = absLogDir.ToURI().ToString();
            Configuration conf           = new Configuration();

            conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
            NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();

            healthChecker.Init(conf);
            LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();

            NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
                                                                        , new ApplicationACLsManager(conf), new NMNullStateStoreService());
            // Add an application and the corresponding containers
            RecordFactory recordFactory    = RecordFactoryProvider.GetRecordFactory(conf);
            string        user             = "******";
            long          clusterTimeStamp = 1234;
            ApplicationId appId            = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
                                                                           , 1);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                >();
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                 , 0);

            nmContext.GetApplications()[appId] = app;
            MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
                                                        conf, user, appId, 1);

            container.SetState(ContainerState.Running);
            nmContext.GetContainers()[container1] = container;
            IList <FilePath> files = null;

            files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
            NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
            // After container is completed, it is removed from nmContext
            Sharpen.Collections.Remove(nmContext.GetContainers(), container1);
            NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]);
            files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
            NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
            // Create a new context to check if correct container log dirs are fetched
            // on full disk.
            LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler
                                                                                     );

            // good log dirs are empty and nm log dir is in the full log dir list.
            Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList
                                                                                     <string>());
            Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays
                                                                                            .AsList(new string[] { absLogDir.GetAbsolutePath() }));
            nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager
                                                      (conf), new NMNullStateStoreService());
            nmContext.GetApplications()[appId] = app;
            container.SetState(ContainerState.Running);
            nmContext.GetContainers()[container1] = container;
            IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext
                                                                           );
            FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1);

            NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir));
        }
Esempio n. 23
0
        /// <exception cref="System.Exception"/>
        public virtual void TestContainerUpdate()
        {
            //Start the node
            node.Handle(new RMNodeStartedEvent(null, null, null));
            NodeId     nodeId = BuilderUtils.NewNodeId("localhost:1", 1);
            RMNodeImpl node2  = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null
                                               );

            node2.Handle(new RMNodeStartedEvent(null, null, null));
            ContainerId completedContainerIdFromNode1 = BuilderUtils.NewContainerId(BuilderUtils
                                                                                    .NewApplicationAttemptId(BuilderUtils.NewApplicationId(0, 0), 0), 0);
            ContainerId completedContainerIdFromNode2_1 = BuilderUtils.NewContainerId(BuilderUtils
                                                                                      .NewApplicationAttemptId(BuilderUtils.NewApplicationId(1, 1), 1), 1);
            ContainerId completedContainerIdFromNode2_2 = BuilderUtils.NewContainerId(BuilderUtils
                                                                                      .NewApplicationAttemptId(BuilderUtils.NewApplicationId(1, 1), 1), 2);
            RMNodeStatusEvent statusEventFromNode1     = GetMockRMNodeStatusEvent();
            RMNodeStatusEvent statusEventFromNode2_1   = GetMockRMNodeStatusEvent();
            RMNodeStatusEvent statusEventFromNode2_2   = GetMockRMNodeStatusEvent();
            ContainerStatus   containerStatusFromNode1 = Org.Mockito.Mockito.Mock <ContainerStatus
                                                                                   >();
            ContainerStatus containerStatusFromNode2_1 = Org.Mockito.Mockito.Mock <ContainerStatus
                                                                                   >();
            ContainerStatus containerStatusFromNode2_2 = Org.Mockito.Mockito.Mock <ContainerStatus
                                                                                   >();

            Org.Mockito.Mockito.DoReturn(completedContainerIdFromNode1).When(containerStatusFromNode1
                                                                             ).GetContainerId();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatusFromNode1
                                                                           )).When(statusEventFromNode1).GetContainers();
            node.Handle(statusEventFromNode1);
            NUnit.Framework.Assert.AreEqual(1, completedContainers.Count);
            NUnit.Framework.Assert.AreEqual(completedContainerIdFromNode1, completedContainers
                                            [0].GetContainerId());
            completedContainers.Clear();
            Org.Mockito.Mockito.DoReturn(completedContainerIdFromNode2_1).When(containerStatusFromNode2_1
                                                                               ).GetContainerId();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatusFromNode2_1
                                                                           )).When(statusEventFromNode2_1).GetContainers();
            Org.Mockito.Mockito.DoReturn(completedContainerIdFromNode2_2).When(containerStatusFromNode2_2
                                                                               ).GetContainerId();
            Org.Mockito.Mockito.DoReturn(Sharpen.Collections.SingletonList(containerStatusFromNode2_2
                                                                           )).When(statusEventFromNode2_2).GetContainers();
            node2.SetNextHeartBeat(false);
            node2.Handle(statusEventFromNode2_1);
            node2.SetNextHeartBeat(true);
            node2.Handle(statusEventFromNode2_2);
            NUnit.Framework.Assert.AreEqual(2, completedContainers.Count);
            NUnit.Framework.Assert.AreEqual(completedContainerIdFromNode2_1, completedContainers
                                            [0].GetContainerId());
            NUnit.Framework.Assert.AreEqual(completedContainerIdFromNode2_2, completedContainers
                                            [1].GetContainerId());
        }
Esempio n. 24
0
        public virtual void TestExpireWhileRunning()
        {
            DrainDispatcher drainDispatcher = new DrainDispatcher();
            EventHandler <RMAppAttemptEvent> appAttemptEventHandler = Org.Mockito.Mockito.Mock
                                                                      <EventHandler>();
            EventHandler generic = Org.Mockito.Mockito.Mock <EventHandler>();

            drainDispatcher.Register(typeof(RMAppAttemptEventType), appAttemptEventHandler);
            drainDispatcher.Register(typeof(RMNodeEventType), generic);
            drainDispatcher.Init(new YarnConfiguration());
            drainDispatcher.Start();
            NodeId               nodeId       = BuilderUtils.NewNodeId("host", 3425);
            ApplicationId        appId        = BuilderUtils.NewApplicationId(1, 1);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId containerId            = BuilderUtils.NewContainerId(appAttemptId, 1);
            ContainerAllocationExpirer expirer = Org.Mockito.Mockito.Mock <ContainerAllocationExpirer
                                                                           >();
            Resource  resource  = BuilderUtils.NewResource(512, 1);
            Priority  priority  = BuilderUtils.NewPriority(5);
            Container container = BuilderUtils.NewContainer(containerId, nodeId, "host:3465",
                                                            resource, priority, null);
            RMApplicationHistoryWriter writer = Org.Mockito.Mockito.Mock <RMApplicationHistoryWriter
                                                                          >();
            SystemMetricsPublisher publisher = Org.Mockito.Mockito.Mock <SystemMetricsPublisher
                                                                         >();
            RMContext rmContext = Org.Mockito.Mockito.Mock <RMContext>();

            Org.Mockito.Mockito.When(rmContext.GetDispatcher()).ThenReturn(drainDispatcher);
            Org.Mockito.Mockito.When(rmContext.GetContainerAllocationExpirer()).ThenReturn(expirer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetRMApplicationHistoryWriter()).ThenReturn(writer
                                                                                           );
            Org.Mockito.Mockito.When(rmContext.GetSystemMetricsPublisher()).ThenReturn(publisher
                                                                                       );
            Org.Mockito.Mockito.When(rmContext.GetYarnConfiguration()).ThenReturn(new YarnConfiguration
                                                                                      ());
            RMContainer rmContainer = new RMContainerImpl(container, appAttemptId, nodeId, "user"
                                                          , rmContext);

            NUnit.Framework.Assert.AreEqual(RMContainerState.New, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual(resource, rmContainer.GetAllocatedResource());
            NUnit.Framework.Assert.AreEqual(nodeId, rmContainer.GetAllocatedNode());
            NUnit.Framework.Assert.AreEqual(priority, rmContainer.GetAllocatedPriority());
            Org.Mockito.Mockito.Verify(writer).ContainerStarted(Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher).ContainerCreated(Matchers.Any <RMContainer>(
                                                                       ), Matchers.AnyLong());
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Start));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Allocated, rmContainer.GetState(
                                                ));
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Acquired
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Acquired, rmContainer.GetState()
                                            );
            rmContainer.Handle(new RMContainerEvent(containerId, RMContainerEventType.Launched
                                                    ));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            NUnit.Framework.Assert.AreEqual("http://host:3465/node/containerlogs/container_1_0001_01_000001/user"
                                            , rmContainer.GetLogURL());
            // In RUNNING state. Verify EXPIRE and associated actions.
            Org.Mockito.Mockito.Reset(appAttemptEventHandler);
            ContainerStatus containerStatus = SchedulerUtils.CreateAbnormalContainerStatus(containerId
                                                                                           , SchedulerUtils.ExpiredContainer);

            rmContainer.Handle(new RMContainerFinishedEvent(containerId, containerStatus, RMContainerEventType
                                                            .Expire));
            drainDispatcher.Await();
            NUnit.Framework.Assert.AreEqual(RMContainerState.Running, rmContainer.GetState());
            Org.Mockito.Mockito.Verify(writer, Org.Mockito.Mockito.Never()).ContainerFinished
                (Matchers.Any <RMContainer>());
            Org.Mockito.Mockito.Verify(publisher, Org.Mockito.Mockito.Never()).ContainerFinished
                (Matchers.Any <RMContainer>(), Matchers.AnyLong());
        }
Esempio n. 25
0
        /// <exception cref="System.Exception"/>
        public virtual void TestHeadroom()
        {
            Configuration conf = new Configuration();

            conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
                                                                                       ));
            MockRM rm = new MockRM(conf);

            rm.Start();
            FifoScheduler fs = (FifoScheduler)rm.GetResourceScheduler();
            // Add a node
            RMNode n1 = MockNodes.NewNodeInfo(0, MockNodes.NewResource(4 * Gb), 1, "127.0.0.2"
                                              );

            fs.Handle(new NodeAddedSchedulerEvent(n1));
            // Add two applications
            ApplicationId        appId1        = BuilderUtils.NewApplicationId(100, 1);
            ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1,
                                                                                      1);

            CreateMockRMApp(appAttemptId1, rm.GetRMContext());
            SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");

            fs.Handle(appEvent);
            SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false
                                                                            );

            fs.Handle(attemptEvent);
            ApplicationId        appId2        = BuilderUtils.NewApplicationId(200, 2);
            ApplicationAttemptId appAttemptId2 = BuilderUtils.NewApplicationAttemptId(appId2,
                                                                                      1);

            CreateMockRMApp(appAttemptId2, rm.GetRMContext());
            SchedulerEvent appEvent2 = new AppAddedSchedulerEvent(appId2, "queue", "user");

            fs.Handle(appEvent2);
            SchedulerEvent attemptEvent2 = new AppAttemptAddedSchedulerEvent(appAttemptId2, false
                                                                             );

            fs.Handle(attemptEvent2);
            IList <ContainerId>     emptyId  = new AList <ContainerId>();
            IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>();
            // Set up resource requests
            // Ask for a 1 GB container for app 1
            IList <ResourceRequest> ask1 = new AList <ResourceRequest>();

            ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
                                                         .Any, BuilderUtils.NewResource(Gb, 1), 1));
            fs.Allocate(appAttemptId1, ask1, emptyId, null, null);
            // Ask for a 2 GB container for app 2
            IList <ResourceRequest> ask2 = new AList <ResourceRequest>();

            ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
                                                         .Any, BuilderUtils.NewResource(2 * Gb, 1), 1));
            fs.Allocate(appAttemptId2, ask2, emptyId, null, null);
            // Trigger container assignment
            fs.Handle(new NodeUpdateSchedulerEvent(n1));
            // Get the allocation for the applications and verify headroom
            Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation1.GetResourceLimit
                                                ().GetMemory());
            Allocation allocation2 = fs.Allocate(appAttemptId2, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("Allocation headroom", 1 * Gb, allocation2.GetResourceLimit
                                                ().GetMemory());
            rm.Stop();
        }
Esempio n. 26
0
        /// <exception cref="System.Exception"/>
        public virtual void TestBlackListNodes()
        {
            Configuration conf = new Configuration();

            conf.SetClass(YarnConfiguration.RmScheduler, typeof(FifoScheduler), typeof(ResourceScheduler
                                                                                       ));
            MockRM rm = new MockRM(conf);

            rm.Start();
            FifoScheduler fs         = (FifoScheduler)rm.GetResourceScheduler();
            int           rack_num_0 = 0;
            int           rack_num_1 = 1;
            // Add 4 nodes in 2 racks
            // host_0_0 in rack0
            string host_0_0 = "127.0.0.1";
            RMNode n1       = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_0
                                                    );

            fs.Handle(new NodeAddedSchedulerEvent(n1));
            // host_0_1 in rack0
            string host_0_1 = "127.0.0.2";
            RMNode n2       = MockNodes.NewNodeInfo(rack_num_0, MockNodes.NewResource(4 * Gb), 1, host_0_1
                                                    );

            fs.Handle(new NodeAddedSchedulerEvent(n2));
            // host_1_0 in rack1
            string host_1_0 = "127.0.0.3";
            RMNode n3       = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_0
                                                    );

            fs.Handle(new NodeAddedSchedulerEvent(n3));
            // host_1_1 in rack1
            string host_1_1 = "127.0.0.4";
            RMNode n4       = MockNodes.NewNodeInfo(rack_num_1, MockNodes.NewResource(4 * Gb), 1, host_1_1
                                                    );

            fs.Handle(new NodeAddedSchedulerEvent(n4));
            // Add one application
            ApplicationId        appId1        = BuilderUtils.NewApplicationId(100, 1);
            ApplicationAttemptId appAttemptId1 = BuilderUtils.NewApplicationAttemptId(appId1,
                                                                                      1);

            CreateMockRMApp(appAttemptId1, rm.GetRMContext());
            SchedulerEvent appEvent = new AppAddedSchedulerEvent(appId1, "queue", "user");

            fs.Handle(appEvent);
            SchedulerEvent attemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId1, false
                                                                            );

            fs.Handle(attemptEvent);
            IList <ContainerId>     emptyId  = new AList <ContainerId>();
            IList <ResourceRequest> emptyAsk = new AList <ResourceRequest>();
            // Allow rack-locality for rack_1, but blacklist host_1_0
            // Set up resource requests
            // Ask for a 1 GB container for app 1
            IList <ResourceRequest> ask1 = new AList <ResourceRequest>();

            ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), "rack1"
                                                         , BuilderUtils.NewResource(Gb, 1), 1));
            ask1.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
                                                         .Any, BuilderUtils.NewResource(Gb, 1), 1));
            fs.Allocate(appAttemptId1, ask1, emptyId, Sharpen.Collections.SingletonList(host_1_0
                                                                                        ), null);
            // Trigger container assignment
            fs.Handle(new NodeUpdateSchedulerEvent(n3));
            // Get the allocation for the application and verify no allocation on blacklist node
            Allocation allocation1 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation1", 0, allocation1.GetContainers().Count
                                            );
            // verify host_1_1 can get allocated as not in blacklist
            fs.Handle(new NodeUpdateSchedulerEvent(n4));
            Allocation allocation2 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation2", 1, allocation2.GetContainers().Count
                                            );
            IList <Container> containerList = allocation2.GetContainers();

            foreach (Container container in containerList)
            {
                NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container.GetNodeId
                                                    (), n4.GetNodeID());
            }
            // Ask for a 1 GB container again for app 1
            IList <ResourceRequest> ask2 = new AList <ResourceRequest>();

            // this time, rack0 is also in blacklist, so only host_1_1 is available to
            // be assigned
            ask2.AddItem(BuilderUtils.NewResourceRequest(BuilderUtils.NewPriority(0), ResourceRequest
                                                         .Any, BuilderUtils.NewResource(Gb, 1), 1));
            fs.Allocate(appAttemptId1, ask2, emptyId, Sharpen.Collections.SingletonList("rack0"
                                                                                        ), null);
            // verify n1 is not qualified to be allocated
            fs.Handle(new NodeUpdateSchedulerEvent(n1));
            Allocation allocation3 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation3", 0, allocation3.GetContainers().Count
                                            );
            // verify n2 is not qualified to be allocated
            fs.Handle(new NodeUpdateSchedulerEvent(n2));
            Allocation allocation4 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation4", 0, allocation4.GetContainers().Count
                                            );
            // verify n3 is not qualified to be allocated
            fs.Handle(new NodeUpdateSchedulerEvent(n3));
            Allocation allocation5 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation5", 0, allocation5.GetContainers().Count
                                            );
            fs.Handle(new NodeUpdateSchedulerEvent(n4));
            Allocation allocation6 = fs.Allocate(appAttemptId1, emptyAsk, emptyId, null, null
                                                 );

            NUnit.Framework.Assert.AreEqual("allocation6", 1, allocation6.GetContainers().Count
                                            );
            containerList = allocation6.GetContainers();
            foreach (Container container_1 in containerList)
            {
                NUnit.Framework.Assert.AreEqual("Container is allocated on n4", container_1.GetNodeId
                                                    (), n4.GetNodeID());
            }
            rm.Stop();
        }