protected internal LocalWrapperScriptBuilder(DefaultContainerExecutor _enclosing,
                                              Path containerWorkDir)
 {
     this._enclosing        = _enclosing;
     this.wrapperScriptPath = new Path(containerWorkDir, Shell.AppendScriptExtension("default_container_executor"
                                                                                     ));
 }
 public WindowsLocalWrapperScriptBuilder(DefaultContainerExecutor _enclosing, string
                                         containerIdStr, Path containerWorkDir)
     : base(_enclosing)
 {
     this._enclosing     = _enclosing;
     this.containerIdStr = containerIdStr;
 }
 public UnixLocalWrapperScriptBuilder(DefaultContainerExecutor _enclosing, Path containerWorkDir
                                      )
     : base(_enclosing)
 {
     this._enclosing        = _enclosing;
     this.sessionScriptPath = new Path(containerWorkDir, Shell.AppendScriptExtension("default_container_executor_session"
                                                                                     ));
 }
Example #4
0
        public virtual void TestKillContainersOnShutdown()
        {
            nm = new TestNodeManagerShutdown.TestNodeManager(this);
            nm.Init(CreateNMConfig());
            nm.Start();
            StartContainer(nm, cId, localFS, tmpDir, processStartFile);
            int MaxTries = 20;
            int numTries = 0;

            while (!processStartFile.Exists() && numTries < MaxTries)
            {
                try
                {
                    Sharpen.Thread.Sleep(500);
                }
                catch (Exception ex)
                {
                    Sharpen.Runtime.PrintStackTrace(ex);
                }
                numTries++;
            }
            nm.Stop();
            // Now verify the contents of the file.  Script generates a message when it
            // receives a sigterm so we look for that.  We cannot perform this check on
            // Windows, because the process is not notified when killed by winutils.
            // There is no way for the process to trap and respond.  Instead, we can
            // verify that the job object with ID matching container ID no longer exists.
            if (Shell.Windows)
            {
                NUnit.Framework.Assert.IsFalse("Process is still alive!", DefaultContainerExecutor
                                               .ContainerIsAlive(cId.ToString()));
            }
            else
            {
                BufferedReader reader = new BufferedReader(new FileReader(processStartFile));
                bool           foundSigTermMessage = false;
                while (true)
                {
                    string line = reader.ReadLine();
                    if (line == null)
                    {
                        break;
                    }
                    if (line.Contains("SIGTERM"))
                    {
                        foundSigTermMessage = true;
                        break;
                    }
                }
                NUnit.Framework.Assert.IsTrue("Did not find sigterm message", foundSigTermMessage
                                              );
                reader.Close();
            }
        }
Example #5
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public virtual void TestStartLocalizer()
        {
            IPEndPoint     localizationServerAddress;
            Path           firstDir  = new Path(BaseTmpPath, "localDir1");
            IList <string> localDirs = new AList <string>();
            Path           secondDir = new Path(BaseTmpPath, "localDir2");
            IList <string> logDirs   = new AList <string>();
            Path           logDir    = new Path(BaseTmpPath, "logDir");
            Path           tokenDir  = new Path(BaseTmpPath, "tokenDir");
            FsPermission   perms     = new FsPermission((short)0x1f8);
            Configuration  conf      = new Configuration();

            localizationServerAddress = conf.GetSocketAddr(YarnConfiguration.NmBindHost, YarnConfiguration
                                                           .NmLocalizerAddress, YarnConfiguration.DefaultNmLocalizerAddress, YarnConfiguration
                                                           .DefaultNmLocalizerPort);
            FileContext mockLfs = Org.Mockito.Mockito.Spy(FileContext.GetLocalFSFileContext(conf
                                                                                            ));

            FileContext.Util mockUtil = Org.Mockito.Mockito.Spy(mockLfs.Util());
            Org.Mockito.Mockito.DoAnswer(new _Answer_344(mockUtil)).When(mockLfs).Util();
            Org.Mockito.Mockito.DoAnswer(new _Answer_351(firstDir, mockLfs)).When(mockUtil).Copy
                (Matchers.Any <Path>(), Matchers.Any <Path>());
            // throw an Exception when copy token to the first local dir
            // to simulate no space on the first drive
            // copy token to the second local dir
            Org.Mockito.Mockito.DoAnswer(new _Answer_378(firstDir)).When(mockLfs).GetFsStatus
                (Matchers.Any <Path>());
            // let second local directory return more free space than
            // first local directory
            DefaultContainerExecutor mockExec = Org.Mockito.Mockito.Spy(new DefaultContainerExecutor
                                                                            (mockLfs));

            mockExec.SetConf(conf);
            localDirs.AddItem(mockLfs.MakeQualified(firstDir).ToString());
            localDirs.AddItem(mockLfs.MakeQualified(secondDir).ToString());
            logDirs.AddItem(mockLfs.MakeQualified(logDir).ToString());
            conf.SetStrings(YarnConfiguration.NmLocalDirs, Sharpen.Collections.ToArray(localDirs
                                                                                       , new string[localDirs.Count]));
            conf.Set(YarnConfiguration.NmLogDirs, logDir.ToString());
            mockLfs.Mkdir(tokenDir, perms, true);
            Path   nmPrivateCTokensPath = new Path(tokenDir, "test.tokens");
            string appSubmitter         = "nobody";
            string appId = "APP_ID";
            string locId = "LOC_ID";
            LocalDirsHandlerService dirsHandler = Org.Mockito.Mockito.Mock <LocalDirsHandlerService
                                                                            >();

            Org.Mockito.Mockito.When(dirsHandler.GetLocalDirs()).ThenReturn(localDirs);
            Org.Mockito.Mockito.When(dirsHandler.GetLogDirs()).ThenReturn(logDirs);
            try
            {
                mockExec.StartLocalizer(nmPrivateCTokensPath, localizationServerAddress, appSubmitter
                                        , appId, locId, dirsHandler);
            }
            catch (IOException e)
            {
                NUnit.Framework.Assert.Fail("StartLocalizer failed to copy token file " + e);
            }
            finally
            {
                mockExec.DeleteAsUser(appSubmitter, firstDir);
                mockExec.DeleteAsUser(appSubmitter, secondDir);
                mockExec.DeleteAsUser(appSubmitter, logDir);
                DeleteTmpFiles();
            }
        }
Example #6
0
        public virtual void TestContainerLaunchError()
        {
            if (Shell.Windows)
            {
                BaseTmpPath = new Path(new FilePath("target").GetAbsolutePath(), typeof(TestDefaultContainerExecutor
                                                                                        ).Name);
            }
            Path           localDir  = new Path(BaseTmpPath, "localDir");
            IList <string> localDirs = new AList <string>();

            localDirs.AddItem(localDir.ToString());
            IList <string> logDirs = new AList <string>();
            Path           logDir  = new Path(BaseTmpPath, "logDir");

            logDirs.AddItem(logDir.ToString());
            Configuration conf = new Configuration();

            conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
            conf.Set(YarnConfiguration.NmLocalDirs, localDir.ToString());
            conf.Set(YarnConfiguration.NmLogDirs, logDir.ToString());
            FileContext lfs = FileContext.GetLocalFSFileContext(conf);
            DefaultContainerExecutor mockExec = Org.Mockito.Mockito.Spy(new DefaultContainerExecutor
                                                                            (lfs));

            mockExec.SetConf(conf);
            Org.Mockito.Mockito.DoAnswer(new _Answer_245()).When(mockExec).LogOutput(Matchers.Any
                                                                                     <string>());
            string appSubmitter = "nobody";
            string appId        = "APP_ID";
            string containerId  = "CONTAINER_ID";

            Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container container
                = Org.Mockito.Mockito.Mock <Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                            >();
            ContainerId            cId     = Org.Mockito.Mockito.Mock <ContainerId>();
            ContainerLaunchContext context = Org.Mockito.Mockito.Mock <ContainerLaunchContext>
                                                 ();
            Dictionary <string, string> env = new Dictionary <string, string>();

            Org.Mockito.Mockito.When(container.GetContainerId()).ThenReturn(cId);
            Org.Mockito.Mockito.When(container.GetLaunchContext()).ThenReturn(context);
            try
            {
                Org.Mockito.Mockito.DoAnswer(new _Answer_268()).When(container).Handle(Matchers.Any
                                                                                       <ContainerDiagnosticsUpdateEvent>());
                Org.Mockito.Mockito.When(cId.ToString()).ThenReturn(containerId);
                Org.Mockito.Mockito.When(cId.GetApplicationAttemptId()).ThenReturn(ApplicationAttemptId
                                                                                   .NewInstance(ApplicationId.NewInstance(0, 1), 0));
                Org.Mockito.Mockito.When(context.GetEnvironment()).ThenReturn(env);
                mockExec.CreateUserLocalDirs(localDirs, appSubmitter);
                mockExec.CreateUserCacheDirs(localDirs, appSubmitter);
                mockExec.CreateAppDirs(localDirs, appSubmitter, appId);
                mockExec.CreateAppLogDirs(appId, logDirs, appSubmitter);
                Path scriptPath = new Path("file:///bin/echo");
                Path tokensPath = new Path("file:///dev/null");
                if (Shell.Windows)
                {
                    FilePath       tmp    = new FilePath(BaseTmpPath.ToString(), "test_echo.cmd");
                    BufferedWriter output = new BufferedWriter(new FileWriter(tmp));
                    output.Write("Exit 1");
                    output.Write("Echo No such file or directory 1>&2");
                    output.Close();
                    scriptPath = new Path(tmp.GetAbsolutePath());
                    tmp        = new FilePath(BaseTmpPath.ToString(), "tokens");
                    tmp.CreateNewFile();
                    tokensPath = new Path(tmp.GetAbsolutePath());
                }
                Path workDir = localDir;
                Path pidFile = new Path(workDir, "pid.txt");
                mockExec.Init();
                mockExec.ActivateContainer(cId, pidFile);
                int ret = mockExec.LaunchContainer(container, scriptPath, tokensPath, appSubmitter
                                                   , appId, workDir, localDirs, localDirs);
                NUnit.Framework.Assert.AreNotSame(0, ret);
            }
            finally
            {
                mockExec.DeleteAsUser(appSubmitter, localDir);
                mockExec.DeleteAsUser(appSubmitter, logDir);
            }
        }
Example #7
0
        public virtual void TestDirPermissions()
        {
            DeleteTmpFiles();
            string       user          = "******";
            string       appId         = "app_12345_123";
            FsPermission userCachePerm = new FsPermission(DefaultContainerExecutor.UserPerm);
            FsPermission appCachePerm  = new FsPermission(DefaultContainerExecutor.AppcachePerm
                                                          );
            FsPermission fileCachePerm = new FsPermission(DefaultContainerExecutor.FilecachePerm
                                                          );
            FsPermission   appDirPerm = new FsPermission(DefaultContainerExecutor.AppdirPerm);
            FsPermission   logDirPerm = new FsPermission(DefaultContainerExecutor.LogdirPerm);
            IList <string> localDirs  = new AList <string>();

            localDirs.AddItem(new Path(BaseTmpPath, "localDirA").ToString());
            localDirs.AddItem(new Path(BaseTmpPath, "localDirB").ToString());
            IList <string> logDirs = new AList <string>();

            logDirs.AddItem(new Path(BaseTmpPath, "logDirA").ToString());
            logDirs.AddItem(new Path(BaseTmpPath, "logDirB").ToString());
            Configuration conf = new Configuration();

            conf.Set(CommonConfigurationKeys.FsPermissionsUmaskKey, "077");
            FileContext lfs = FileContext.GetLocalFSFileContext(conf);
            DefaultContainerExecutor executor = new DefaultContainerExecutor(lfs);

            executor.Init();
            try
            {
                executor.CreateUserLocalDirs(localDirs, user);
                executor.CreateUserCacheDirs(localDirs, user);
                executor.CreateAppDirs(localDirs, user, appId);
                foreach (string dir in localDirs)
                {
                    FileStatus stats = lfs.GetFileStatus(new Path(new Path(dir, ContainerLocalizer.Usercache
                                                                           ), user));
                    NUnit.Framework.Assert.AreEqual(userCachePerm, stats.GetPermission());
                }
                foreach (string dir_1 in localDirs)
                {
                    Path userCachePath = new Path(new Path(dir_1, ContainerLocalizer.Usercache), user
                                                  );
                    Path       appCachePath = new Path(userCachePath, ContainerLocalizer.Appcache);
                    FileStatus stats        = lfs.GetFileStatus(appCachePath);
                    NUnit.Framework.Assert.AreEqual(appCachePerm, stats.GetPermission());
                    stats = lfs.GetFileStatus(new Path(userCachePath, ContainerLocalizer.Filecache));
                    NUnit.Framework.Assert.AreEqual(fileCachePerm, stats.GetPermission());
                    stats = lfs.GetFileStatus(new Path(appCachePath, appId));
                    NUnit.Framework.Assert.AreEqual(appDirPerm, stats.GetPermission());
                }
                executor.CreateAppLogDirs(appId, logDirs, user);
                foreach (string dir_2 in logDirs)
                {
                    FileStatus stats = lfs.GetFileStatus(new Path(dir_2, appId));
                    NUnit.Framework.Assert.AreEqual(logDirPerm, stats.GetPermission());
                }
            }
            finally
            {
                DeleteTmpFiles();
            }
        }
Example #8
0
        public virtual void TestSuccessfulContainerLaunch()
        {
            FileContext localFS = FileContext.GetLocalFSFileContext();

            localFS.Delete(new Path(localDir.GetAbsolutePath()), true);
            localFS.Delete(new Path(localLogDir.GetAbsolutePath()), true);
            localFS.Delete(new Path(remoteLogDir.GetAbsolutePath()), true);
            localDir.Mkdir();
            localLogDir.Mkdir();
            remoteLogDir.Mkdir();
            YarnConfiguration conf    = new YarnConfiguration();
            Context           context = new _NMContext_84(new NMContainerTokenSecretManager(conf), new
                                                          NMTokenSecretManagerInNM(), null, null, new NMNullStateStoreService());

            conf.Set(YarnConfiguration.NmLocalDirs, localDir.GetAbsolutePath());
            conf.Set(YarnConfiguration.NmLogDirs, localLogDir.GetAbsolutePath());
            conf.Set(YarnConfiguration.NmRemoteAppLogDir, remoteLogDir.GetAbsolutePath());
            ContainerExecutor exec = new DefaultContainerExecutor();

            exec.SetConf(conf);
            DeletionService          del           = new DeletionService(exec);
            Dispatcher               dispatcher    = new AsyncDispatcher();
            NodeHealthCheckerService healthChecker = new NodeHealthCheckerService();

            healthChecker.Init(conf);
            LocalDirsHandlerService dirsHandler       = healthChecker.GetDiskHandler();
            NodeManagerMetrics      metrics           = NodeManagerMetrics.Create();
            NodeStatusUpdater       nodeStatusUpdater = new _NodeStatusUpdaterImpl_106(context, dispatcher
                                                                                       , healthChecker, metrics);
            // Don't start any updating thread.
            DummyContainerManager containerManager = new DummyContainerManager(context, exec,
                                                                               del, nodeStatusUpdater, metrics, new ApplicationACLsManager(conf), dirsHandler);

            nodeStatusUpdater.Init(conf);
            ((NodeManager.NMContext)context).SetContainerManager(containerManager);
            nodeStatusUpdater.Start();
            containerManager.Init(conf);
            containerManager.Start();
            ContainerLaunchContext launchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
                                                                                    >();
            ApplicationId        applicationId        = ApplicationId.NewInstance(0, 0);
            ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
                                                                                         , 0);
            ContainerId           cID       = ContainerId.NewContainerId(applicationAttemptId, 0);
            string                user      = "******";
            StartContainerRequest scRequest = StartContainerRequest.NewInstance(launchContext
                                                                                , TestContainerManager.CreateContainerToken(cID, SimulatedRmIdentifier, context.
                                                                                                                            GetNodeId(), user, context.GetContainerTokenSecretManager()));
            IList <StartContainerRequest> list = new AList <StartContainerRequest>();

            list.AddItem(scRequest);
            StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);

            containerManager.StartContainers(allRequests);
            BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState
                                                           .Running);
            IList <ContainerId> containerIds = new AList <ContainerId>();

            containerIds.AddItem(cID);
            StopContainersRequest stopRequest = StopContainersRequest.NewInstance(containerIds
                                                                                  );

            containerManager.StopContainers(stopRequest);
            BaseContainerManagerTest.WaitForContainerState(containerManager, cID, ContainerState
                                                           .Complete);
            containerManager.Stop();
        }