Example #1
0
        public virtual void Setup()
        {
            localFS.Delete(new Path(localDir.GetAbsolutePath()), true);
            localFS.Delete(new Path(tmpDir.GetAbsolutePath()), true);
            localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true);
            localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true);
            localDir.Mkdir();
            tmpDir.Mkdir();
            localLogDir.Mkdir();
            remoteLogDir.Mkdir();
            Log.Info("Created localDir in " + localDir.GetAbsolutePath());
            Log.Info("Created tmpDir in " + tmpDir.GetAbsolutePath());
            string bindAddress = "0.0.0.0:12345";

            conf.Set(YarnConfiguration.NmAddress, bindAddress);
            conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
            conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath());
            conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath());
            conf.SetLong(YarnConfiguration.NmLogRetainSeconds, 1);
            // Default delSrvc
            delSrvc = CreateDeletionService();
            delSrvc.Init(conf);
            exec = CreateContainerExecutor();
            nodeHealthChecker = new NodeHealthCheckerService();
            nodeHealthChecker.Init(conf);
            dirsHandler      = nodeHealthChecker.GetDiskHandler();
            containerManager = CreateContainerManager(delSrvc);
            ((NodeManager.NMContext)context).SetContainerManager(containerManager);
            nodeStatusUpdater.Init(conf);
            containerManager.Init(conf);
            nodeStatusUpdater.Start();
        }
Example #2
0
 public _NodeStatusUpdaterImpl_121(BaseContainerManagerTest _enclosing, Context baseArg1
                                   , Dispatcher baseArg2, NodeHealthCheckerService baseArg3, NodeManagerMetrics baseArg4
                                   )
     : base(baseArg1, baseArg2, baseArg3, baseArg4)
 {
     this._enclosing = _enclosing;
 }
Example #3
0
 public _NodeStatusUpdaterImpl_601(ShortCircuitedNodeManager _enclosing, Context baseArg1
                                   , Dispatcher baseArg2, NodeHealthCheckerService baseArg3, NodeManagerMetrics baseArg4
                                   )
     : base(baseArg1, baseArg2, baseArg3, baseArg4)
 {
     this._enclosing = _enclosing;
 }
Example #4
0
        private int StartNMWebAppServer(string webAddr)
        {
            Context       nmContext    = new NodeManager.NMContext(null, null, null, null, null);
            ResourceView  resourceView = new _ResourceView_84();
            Configuration conf         = new Configuration();

            conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
            conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
            NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();

            healthChecker.Init(conf);
            LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();

            conf.Set(YarnConfiguration.NmWebappAddress, webAddr);
            WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
                                                 (conf), dirsHandler);

            try
            {
                server.Init(conf);
                server.Start();
                return(server.GetPort());
            }
            finally
            {
                server.Stop();
                healthChecker.Stop();
            }
        }
Example #5
0
        public virtual void TestNMWebApp()
        {
            Context       nmContext    = new NodeManager.NMContext(null, null, null, null, null);
            ResourceView  resourceView = new _ResourceView_147();
            Configuration conf         = new Configuration();

            conf.Set(YarnConfiguration.NmLocalDirs, testRootDir.GetAbsolutePath());
            conf.Set(YarnConfiguration.NmLogDirs, testLogDir.GetAbsolutePath());
            NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();

            healthChecker.Init(conf);
            LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
            WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager
                                                 (conf), dirsHandler);

            server.Init(conf);
            server.Start();
            // Add an application and the corresponding containers
            RecordFactory recordFactory    = RecordFactoryProvider.GetRecordFactory(conf);
            Dispatcher    dispatcher       = new AsyncDispatcher();
            string        user             = "******";
            long          clusterTimeStamp = 1234;
            ApplicationId appId            = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
                                                                           , 1);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                >();
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
            nmContext.GetApplications()[appId] = app;
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                 , 0);
            ContainerId container2 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                 , 1);
            NodeManagerMetrics  metrics    = Org.Mockito.Mockito.Mock <NodeManagerMetrics>();
            NMStateStoreService stateStore = new NMNullStateStoreService();

            foreach (ContainerId containerId in new ContainerId[] { container1, container2 })
            {
                // TODO: Use builder utils
                ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                        >();
                long  currentTime    = Runtime.CurrentTimeMillis();
                Token containerToken = BuilderUtils.NewContainerToken(containerId, "127.0.0.1", 1234
                                                                      , user, BuilderUtils.NewResource(1024, 1), currentTime + 10000L, 123, Sharpen.Runtime.GetBytesForString
                                                                          ("password"), currentTime);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
                    = new _ContainerImpl_214(conf, dispatcher, stateStore, launchContext, null, metrics
                                             , BuilderUtils.NewContainerTokenIdentifier(containerToken));
                nmContext.GetContainers()[containerId] = container;
                //TODO: Gross hack. Fix in code.
                ApplicationId applicationId = containerId.GetApplicationAttemptId().GetApplicationId
                                                  ();
                nmContext.GetApplications()[applicationId].GetContainers()[containerId] = container;
                WriteContainerLogs(nmContext, containerId, dirsHandler);
            }
        }
            protected override void ConfigureServlets()
            {
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.conf.Set(YarnConfiguration
                                                                                                .NmLocalDirs, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                                                                .testRootDir.GetAbsolutePath());
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.conf.Set(YarnConfiguration
                                                                                                .NmLogDirs, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                                                                .testLogDir.GetAbsolutePath());
                NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();

                healthChecker.Init(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                   .conf);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.dirsHandler
                    = healthChecker.GetDiskHandler();
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.aclsManager
                    = new ApplicationACLsManager(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                 .conf);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.nmContext
                    = new NodeManager.NMContext(null, null, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                .dirsHandler, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                .aclsManager, null);
                NodeId nodeId = NodeId.NewInstance("testhost.foo.com", 9999);

                ((NodeManager.NMContext)Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                 .nmContext).SetNodeId(nodeId);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.resourceView
                    = new _ResourceView_106();
                // 15.5G in bytes
                // 16G in bytes
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps.nmWebApp =
                    new WebServer.NMWebApp(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                           .resourceView, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                           .aclsManager, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                           .dirsHandler);
                this.Bind <JAXBContextResolver>();
                this.Bind <NMWebServices>();
                this.Bind <GenericExceptionHandler>();
                this.Bind <Context>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                 .nmContext);
                this.Bind <WebApp>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                .nmWebApp);
                this.Bind <ResourceView>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                      .resourceView);
                this.Bind <ApplicationACLsManager>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                                .aclsManager);
                this.Bind <LocalDirsHandlerService>().ToInstance(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Webapp.TestNMWebServicesApps
                                                                 .dirsHandler);
                this.Serve("/*").With(typeof(GuiceContainer));
            }
Example #7
0
 protected override NodeStatusUpdater CreateNodeStatusUpdater(Context context, Dispatcher
                                                              dispatcher, NodeHealthCheckerService healthChecker)
 {
     return(new _NodeStatusUpdaterImpl_601(this, context, dispatcher, healthChecker, this
                                           .metrics));
 }
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Org.Apache.Hadoop.Yarn.Exceptions.YarnException"/>
        public virtual void TestContainerLogDirs()
        {
            FilePath absLogDir = new FilePath("target", typeof(TestNMWebServer).Name + "LogDir"
                                              ).GetAbsoluteFile();
            string        logdirwithFile = absLogDir.ToURI().ToString();
            Configuration conf           = new Configuration();

            conf.Set(YarnConfiguration.NmLogDirs, logdirwithFile);
            NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();

            healthChecker.Init(conf);
            LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();

            NodeManager.NMContext nmContext = new NodeManager.NMContext(null, null, dirsHandler
                                                                        , new ApplicationACLsManager(conf), new NMNullStateStoreService());
            // Add an application and the corresponding containers
            RecordFactory recordFactory    = RecordFactoryProvider.GetRecordFactory(conf);
            string        user             = "******";
            long          clusterTimeStamp = 1234;
            ApplicationId appId            = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
                                                                           , 1);

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                >();
            Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
            Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
            ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                     );
            ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                 , 0);

            nmContext.GetApplications()[appId] = app;
            MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
                                                        conf, user, appId, 1);

            container.SetState(ContainerState.Running);
            nmContext.GetContainers()[container1] = container;
            IList <FilePath> files = null;

            files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
            NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
            // After container is completed, it is removed from nmContext
            Sharpen.Collections.Remove(nmContext.GetContainers(), container1);
            NUnit.Framework.Assert.IsNull(nmContext.GetContainers()[container1]);
            files = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext);
            NUnit.Framework.Assert.IsTrue(!(files[0].ToString().Contains("file:")));
            // Create a new context to check if correct container log dirs are fetched
            // on full disk.
            LocalDirsHandlerService dirsHandlerForFullDisk = Org.Mockito.Mockito.Spy(dirsHandler
                                                                                     );

            // good log dirs are empty and nm log dir is in the full log dir list.
            Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirs()).ThenReturn(new AList
                                                                                     <string>());
            Org.Mockito.Mockito.When(dirsHandlerForFullDisk.GetLogDirsForRead()).ThenReturn(Arrays
                                                                                            .AsList(new string[] { absLogDir.GetAbsolutePath() }));
            nmContext = new NodeManager.NMContext(null, null, dirsHandlerForFullDisk, new ApplicationACLsManager
                                                      (conf), new NMNullStateStoreService());
            nmContext.GetApplications()[appId] = app;
            container.SetState(ContainerState.Running);
            nmContext.GetContainers()[container1] = container;
            IList <FilePath> dirs = ContainerLogsUtils.GetContainerLogDirs(container1, user, nmContext
                                                                           );
            FilePath containerLogDir = new FilePath(absLogDir, appId + "/" + container1);

            NUnit.Framework.Assert.IsTrue(dirs.Contains(containerLogDir));
        }
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestContainerLogPageAccess()
        {
            // SecureIOUtils require Native IO to be enabled. This test will run
            // only if it is enabled.
            Assume.AssumeTrue(NativeIO.IsAvailable());
            string   user         = "******" + Runtime.CurrentTimeMillis();
            FilePath absLogDir    = null;
            FilePath appDir       = null;
            FilePath containerDir = null;
            FilePath syslog       = null;

            try
            {
                // target log directory
                absLogDir = new FilePath("target", typeof(TestContainerLogsPage).Name + "LogDir")
                            .GetAbsoluteFile();
                absLogDir.Mkdir();
                Configuration conf = new Configuration();
                conf.Set(YarnConfiguration.NmLogDirs, absLogDir.ToURI().ToString());
                conf.Set(CommonConfigurationKeysPublic.HadoopSecurityAuthentication, "kerberos");
                UserGroupInformation.SetConfiguration(conf);
                NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();
                healthChecker.Init(conf);
                LocalDirsHandlerService dirsHandler = healthChecker.GetDiskHandler();
                // Add an application and the corresponding containers
                RecordFactory recordFactory    = RecordFactoryProvider.GetRecordFactory(conf);
                long          clusterTimeStamp = 1234;
                ApplicationId appId            = BuilderUtils.NewApplicationId(recordFactory, clusterTimeStamp
                                                                               , 1);
                Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                    app = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                    >();
                Org.Mockito.Mockito.When(app.GetAppId()).ThenReturn(appId);
                // Making sure that application returns a random user. This is required
                // for SecureIOUtils' file owner check.
                Org.Mockito.Mockito.When(app.GetUser()).ThenReturn(user);
                ApplicationAttemptId appAttemptId = BuilderUtils.NewApplicationAttemptId(appId, 1
                                                                                         );
                ContainerId container1 = BuilderUtils.NewContainerId(recordFactory, appId, appAttemptId
                                                                     , 0);
                // Testing secure read access for log files
                // Creating application and container directory and syslog file.
                appDir = new FilePath(absLogDir, appId.ToString());
                appDir.Mkdir();
                containerDir = new FilePath(appDir, container1.ToString());
                containerDir.Mkdir();
                syslog = new FilePath(containerDir, "syslog");
                syslog.CreateNewFile();
                BufferedOutputStream @out = new BufferedOutputStream(new FileOutputStream(syslog)
                                                                     );
                @out.Write(Sharpen.Runtime.GetBytesForString("Log file Content"));
                @out.Close();
                Context context = Org.Mockito.Mockito.Mock <Context>();
                ConcurrentMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                               > appMap = new ConcurrentHashMap <ApplicationId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Application.Application
                                                                 >();
                appMap[appId] = app;
                Org.Mockito.Mockito.When(context.GetApplications()).ThenReturn(appMap);
                ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                   > containers = new ConcurrentHashMap <ContainerId, Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                                                         >();
                Org.Mockito.Mockito.When(context.GetContainers()).ThenReturn(containers);
                Org.Mockito.Mockito.When(context.GetLocalDirsHandler()).ThenReturn(dirsHandler);
                MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(),
                                                            conf, user, appId, 1);
                container.SetState(ContainerState.Running);
                context.GetContainers()[container1] = container;
                ContainerLogsPage.ContainersLogsBlock cLogsBlock = new ContainerLogsPage.ContainersLogsBlock
                                                                       (context);
                IDictionary <string, string> @params = new Dictionary <string, string>();
                @params[YarnWebParams.ContainerId]      = container1.ToString();
                @params[YarnWebParams.ContainerLogType] = "syslog";
                Injector injector = WebAppTests.TestPage <ContainerLogsPage.ContainersLogsBlock>(typeof(
                                                                                                     ContainerLogsPage), cLogsBlock, @params, (Module[])null);
                PrintWriter spyPw = WebAppTests.GetPrintWriter(injector);
                Org.Mockito.Mockito.Verify(spyPw).Write("Exception reading log file. Application submitted by '"
                                                        + user + "' doesn't own requested log file : syslog");
            }
            finally
            {
                if (syslog != null)
                {
                    syslog.Delete();
                }
                if (containerDir != null)
                {
                    containerDir.Delete();
                }
                if (appDir != null)
                {
                    appDir.Delete();
                }
                if (absLogDir != null)
                {
                    absLogDir.Delete();
                }
            }
        }