コード例 #1
0
        /// <exception cref="System.IO.IOException"/>
        public override void Init()
        {
            // Send command to executor which will just start up,
            // verify configuration/permissions and exit
            IList <string> command = new AList <string>(Arrays.AsList(containerExecutorExe, "--checksetup"
                                                                      ));

            string[] commandArray = Sharpen.Collections.ToArray(command, new string[command.Count
                                                                ]);
            Shell.ShellCommandExecutor shExec = new Shell.ShellCommandExecutor(commandArray);
            if (Log.IsDebugEnabled())
            {
                Log.Debug("checkLinuxExecutorSetup: " + Arrays.ToString(commandArray));
            }
            try
            {
                shExec.Execute();
            }
            catch (Shell.ExitCodeException e)
            {
                int exitCode = shExec.GetExitCode();
                Log.Warn("Exit code from container executor initialization is : " + exitCode, e);
                LogOutput(shExec.GetOutput());
                throw new IOException("Linux container executor not configured properly" + " (error="
                                      + exitCode + ")", e);
            }
            resourcesHandler.Init(this);
        }
コード例 #2
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void MountCgroups(IList <string> cgroupKVs, string hierarchy)
        {
            IList <string> command = new AList <string>(Arrays.AsList(containerExecutorExe, "--mount-cgroups"
                                                                      , hierarchy));

            Sharpen.Collections.AddAll(command, cgroupKVs);
            string[] commandArray = Sharpen.Collections.ToArray(command, new string[command.Count
                                                                ]);
            Shell.ShellCommandExecutor shExec = new Shell.ShellCommandExecutor(commandArray);
            if (Log.IsDebugEnabled())
            {
                Log.Debug("mountCgroups: " + Arrays.ToString(commandArray));
            }
            try
            {
                shExec.Execute();
            }
            catch (IOException e)
            {
                int ret_code = shExec.GetExitCode();
                Log.Warn("Exception in LinuxContainerExecutor mountCgroups ", e);
                LogOutput(shExec.GetOutput());
                throw new IOException("Problem mounting cgroups " + cgroupKVs + "; exit code = "
                                      + ret_code + " and output: " + shExec.GetOutput(), e);
            }
        }
コード例 #3
0
        /// <exception cref="System.IO.IOException"/>
        public override bool SignalContainer(string user, string pid, ContainerExecutor.Signal
                                             signal)
        {
            VerifyUsernamePattern(user);
            string runAsUser = GetRunAsUser(user);

            string[] command = new string[] { containerExecutorExe, runAsUser, user, Sharpen.Extensions.ToString
                                                  (LinuxContainerExecutor.Commands.SignalContainer.GetValue()), pid, Sharpen.Extensions.ToString
                                                  (signal.GetValue()) };
            Shell.ShellCommandExecutor shExec = new Shell.ShellCommandExecutor(command);
            if (Log.IsDebugEnabled())
            {
                Log.Debug("signalContainer: " + Arrays.ToString(command));
            }
            try
            {
                shExec.Execute();
            }
            catch (Shell.ExitCodeException e)
            {
                int ret_code = shExec.GetExitCode();
                if (ret_code == LinuxContainerExecutor.ResultCode.InvalidContainerPid.GetValue())
                {
                    return(false);
                }
                Log.Warn("Error in signalling container " + pid + " with " + signal + "; exit = "
                         + ret_code, e);
                LogOutput(shExec.GetOutput());
                throw new IOException("Problem signalling container " + pid + " with " + signal +
                                      "; output: " + shExec.GetOutput() + " and exitCode: " + ret_code, e);
            }
            return(true);
        }
コード例 #4
0
ファイル: ProcessTree.cs プロジェクト: orf53975/hadoop.net
 /// <summary>Send a specified signal to the specified pid</summary>
 /// <param name="pid">the pid of the process [group] to signal.</param>
 /// <param name="signalNum">the signal to send.</param>
 /// <param name="signalName">
 /// the human-readable description of the signal
 /// (for logging).
 /// </param>
 private static void SendSignal(string pid, int signalNum, string signalName)
 {
     Shell.ShellCommandExecutor shexec = null;
     try
     {
         string[] args = new string[] { "kill", "-" + signalNum, pid };
         shexec = new Shell.ShellCommandExecutor(args);
         shexec.Execute();
     }
     catch (IOException ioe)
     {
         Log.Warn("Error executing shell command " + ioe);
     }
     finally
     {
         if (pid.StartsWith("-"))
         {
             Log.Info("Sending signal to all members of process group " + pid + ": " + signalName
                      + ". Exit code " + shexec.GetExitCode());
         }
         else
         {
             Log.Info("Signaling process " + pid + " with " + signalName + ". Exit code " + shexec
                      .GetExitCode());
         }
     }
 }
コード例 #5
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public override void StartLocalizer(Path nmPrivateContainerTokensPath, IPEndPoint
                                            nmAddr, string user, string appId, string locId, LocalDirsHandlerService dirsHandler
                                            )
        {
            IList <string> localDirs = dirsHandler.GetLocalDirs();
            IList <string> logDirs   = dirsHandler.GetLogDirs();

            VerifyUsernamePattern(user);
            string         runAsUser = GetRunAsUser(user);
            IList <string> command   = new AList <string>();

            AddSchedPriorityCommand(command);
            Sharpen.Collections.AddAll(command, Arrays.AsList(containerExecutorExe, runAsUser
                                                              , user, Sharpen.Extensions.ToString(LinuxContainerExecutor.Commands.InitializeContainer
                                                                                                  .GetValue()), appId, nmPrivateContainerTokensPath.ToUri().GetPath().ToString(),
                                                              StringUtils.Join(",", localDirs), StringUtils.Join(",", logDirs)));
            FilePath jvm = new FilePath(new FilePath(Runtime.GetProperty("java.home"), "bin")
                                        , "java");

            // use same jvm as parent
            command.AddItem(jvm.ToString());
            command.AddItem("-classpath");
            command.AddItem(Runtime.GetProperty("java.class.path"));
            string javaLibPath = Runtime.GetProperty("java.library.path");

            if (javaLibPath != null)
            {
                command.AddItem("-Djava.library.path=" + javaLibPath);
            }
            BuildMainArgs(command, user, appId, locId, nmAddr, localDirs);
            string[] commandArray = Sharpen.Collections.ToArray(command, new string[command.Count
                                                                ]);
            Shell.ShellCommandExecutor shExec = new Shell.ShellCommandExecutor(commandArray);
            if (Log.IsDebugEnabled())
            {
                Log.Debug("initApplication: " + Arrays.ToString(commandArray));
            }
            try
            {
                shExec.Execute();
                if (Log.IsDebugEnabled())
                {
                    LogOutput(shExec.GetOutput());
                }
            }
            catch (Shell.ExitCodeException e)
            {
                int exitCode = shExec.GetExitCode();
                Log.Warn("Exit code from container " + locId + " startLocalizer is : " + exitCode
                         , e);
                LogOutput(shExec.GetOutput());
                throw new IOException("Application " + appId + " initialization failed" + " (exitCode="
                                      + exitCode + ") with output: " + shExec.GetOutput(), e);
            }
        }
コード例 #6
0
ファイル: HardLink.cs プロジェクト: orf53975/hadoop.net
        /// <summary>Retrieves the number of links to the specified file.</summary>
        /// <exception cref="System.IO.IOException"/>
        public static int GetLinkCount(FilePath fileName)
        {
            if (fileName == null)
            {
                throw new IOException("invalid argument to getLinkCount: file name is null");
            }
            if (!fileName.Exists())
            {
                throw new FileNotFoundException(fileName + " not found.");
            }
            // construct and execute shell command
            string[]       cmd       = getHardLinkCommand.LinkCount(fileName);
            string         inpMsg    = null;
            string         errMsg    = null;
            int            exitValue = -1;
            BufferedReader @in       = null;

            Shell.ShellCommandExecutor shexec = new Shell.ShellCommandExecutor(cmd);
            try
            {
                shexec.Execute();
                @in       = new BufferedReader(new StringReader(shexec.GetOutput()));
                inpMsg    = @in.ReadLine();
                exitValue = shexec.GetExitCode();
                if (inpMsg == null || exitValue != 0)
                {
                    throw CreateIOException(fileName, inpMsg, errMsg, exitValue, null);
                }
                if (Shell.Solaris)
                {
                    string[] result = inpMsg.Split("\\s+");
                    return(System.Convert.ToInt32(result[1]));
                }
                else
                {
                    return(System.Convert.ToInt32(inpMsg));
                }
            }
            catch (Shell.ExitCodeException e)
            {
                inpMsg    = shexec.GetOutput();
                errMsg    = e.Message;
                exitValue = e.GetExitCode();
                throw CreateIOException(fileName, inpMsg, errMsg, exitValue, e);
            }
            catch (FormatException e)
            {
                throw CreateIOException(fileName, inpMsg, errMsg, exitValue, e);
            }
            finally
            {
                IOUtils.CloseStream(@in);
            }
        }
コード例 #7
0
        public override void DeleteAsUser(string user, Path dir, params Path[] baseDirs)
        {
            VerifyUsernamePattern(user);
            string         runAsUser = GetRunAsUser(user);
            string         dirString = dir == null ? string.Empty : dir.ToUri().GetPath();
            IList <string> command   = new AList <string>(Arrays.AsList(containerExecutorExe, runAsUser
                                                                        , user, Sharpen.Extensions.ToString(LinuxContainerExecutor.Commands.DeleteAsUser
                                                                                                            .GetValue()), dirString));
            IList <string> pathsToDelete = new AList <string>();

            if (baseDirs == null || baseDirs.Length == 0)
            {
                Log.Info("Deleting absolute path : " + dir);
                pathsToDelete.AddItem(dirString);
            }
            else
            {
                foreach (Path baseDir in baseDirs)
                {
                    Path del = dir == null ? baseDir : new Path(baseDir, dir);
                    Log.Info("Deleting path : " + del);
                    pathsToDelete.AddItem(del.ToString());
                    command.AddItem(baseDir.ToUri().GetPath());
                }
            }
            string[] commandArray = Sharpen.Collections.ToArray(command, new string[command.Count
                                                                ]);
            Shell.ShellCommandExecutor shExec = new Shell.ShellCommandExecutor(commandArray);
            if (Log.IsDebugEnabled())
            {
                Log.Debug("deleteAsUser: "******"DeleteAsUser for " + StringUtils.Join(" ", pathsToDelete) + " returned with exit code: "
                          + exitCode, e);
                Log.Error("Output from LinuxContainerExecutor's deleteAsUser follows:");
                LogOutput(shExec.GetOutput());
            }
        }
コード例 #8
0
ファイル: ProcessTree.cs プロジェクト: orf53975/hadoop.net
 /// <summary>
 /// Is the process group with  still alive?
 /// This method assumes that isAlive is called on a pid that was alive not
 /// too long ago, and hence assumes no chance of pid-wrapping-around.
 /// </summary>
 /// <param name="pgrpId">process group id</param>
 /// <returns>true if any of process in group is alive.</returns>
 public static bool IsProcessGroupAlive(string pgrpId)
 {
     Shell.ShellCommandExecutor shexec = null;
     try
     {
         string[] args = new string[] { "kill", "-0", "-" + pgrpId };
         shexec = new Shell.ShellCommandExecutor(args);
         shexec.Execute();
     }
     catch (Shell.ExitCodeException)
     {
         return(false);
     }
     catch (IOException ioe)
     {
         Log.Warn("Error executing shell command " + shexec.ToString() + ioe);
         return(false);
     }
     return(shexec.GetExitCode() == 0 ? true : false);
 }
コード例 #9
0
ファイル: ProcessTree.cs プロジェクト: orf53975/hadoop.net
        private static bool IsSetsidSupported()
        {
            Shell.ShellCommandExecutor shexec = null;
            bool setsidSupported = true;

            try
            {
                string[] args = new string[] { "setsid", "bash", "-c", "echo $$" };
                shexec = new Shell.ShellCommandExecutor(args);
                shexec.Execute();
            }
            catch (IOException)
            {
                Log.Warn("setsid is not available on this machine. So not using it.");
                setsidSupported = false;
            }
            finally
            {
                // handle the exit code
                Log.Info("setsid exited with exit code " + shexec.GetExitCode());
            }
            return(setsidSupported);
        }
コード例 #10
0
        /// <exception cref="System.IO.IOException"/>
        public override int LaunchContainer(Org.Apache.Hadoop.Yarn.Server.Nodemanager.Containermanager.Container.Container
                                            container, Path nmPrivateContainerScriptPath, Path nmPrivateTokensPath, string
                                            userName, string appId, Path containerWorkDir, IList <string> localDirs, IList <string
                                                                                                                            > logDirs)
        {
            string containerImageName = container.GetLaunchContext().GetEnvironment()[YarnConfiguration
                                                                                      .NmDockerContainerExecutorImageName];

            if (Log.IsDebugEnabled())
            {
                Log.Debug("containerImageName from launchContext: " + containerImageName);
            }
            Preconditions.CheckArgument(!Strings.IsNullOrEmpty(containerImageName), "Container image must not be null"
                                        );
            containerImageName = containerImageName.ReplaceAll("['\"]", string.Empty);
            Preconditions.CheckArgument(SaneDockerImage(containerImageName), "Image: " + containerImageName
                                        + " is not a proper docker image");
            string dockerExecutor = GetConf().Get(YarnConfiguration.NmDockerContainerExecutorExecName
                                                  , YarnConfiguration.NmDefaultDockerContainerExecutorExecName);
            FsPermission dirPerm     = new FsPermission(AppdirPerm);
            ContainerId  containerId = container.GetContainerId();
            // create container dirs on all disks
            string containerIdStr = ConverterUtils.ToString(containerId);
            string appIdStr       = ConverterUtils.ToString(containerId.GetApplicationAttemptId().GetApplicationId
                                                                ());

            foreach (string sLocalDir in localDirs)
            {
                Path usersdir     = new Path(sLocalDir, ContainerLocalizer.Usercache);
                Path userdir      = new Path(usersdir, userName);
                Path appCacheDir  = new Path(userdir, ContainerLocalizer.Appcache);
                Path appDir       = new Path(appCacheDir, appIdStr);
                Path containerDir = new Path(appDir, containerIdStr);
                CreateDir(containerDir, dirPerm, true, userName);
            }
            // Create the container log-dirs on all disks
            CreateContainerLogDirs(appIdStr, containerIdStr, logDirs, userName);
            Path tmpDir = new Path(containerWorkDir, YarnConfiguration.DefaultContainerTempDir
                                   );

            CreateDir(tmpDir, dirPerm, false, userName);
            // copy launch script to work dir
            Path launchDst = new Path(containerWorkDir, ContainerLaunch.ContainerScript);

            lfs.Util().Copy(nmPrivateContainerScriptPath, launchDst);
            // copy container tokens to work dir
            Path tokenDst = new Path(containerWorkDir, ContainerLaunch.FinalContainerTokensFile
                                     );

            lfs.Util().Copy(nmPrivateTokensPath, tokenDst);
            string localDirMount         = ToMount(localDirs);
            string logDirMount           = ToMount(logDirs);
            string containerWorkDirMount = ToMount(Sharpen.Collections.SingletonList(containerWorkDir
                                                                                     .ToUri().GetPath()));
            StringBuilder commands   = new StringBuilder();
            string        commandStr = commands.Append(dockerExecutor).Append(" ").Append("run").Append
                                           (" ").Append("--rm --net=host").Append(" ").Append(" --name " + containerIdStr).
                                       Append(localDirMount).Append(logDirMount).Append(containerWorkDirMount).Append(" "
                                                                                                                      ).Append(containerImageName).ToString();
            string dockerPidScript = "`" + dockerExecutor + " inspect --format {{.State.Pid}} "
                                     + containerIdStr + "`";

            // Create new local launch wrapper script
            DockerContainerExecutor.LocalWrapperScriptBuilder sb = new DockerContainerExecutor.UnixLocalWrapperScriptBuilder
                                                                       (this, containerWorkDir, commandStr, dockerPidScript);
            Path pidFile = GetPidFilePath(containerId);

            if (pidFile != null)
            {
                sb.WriteLocalWrapperScript(launchDst, pidFile);
            }
            else
            {
                Log.Info("Container " + containerIdStr + " was marked as inactive. Returning terminated error"
                         );
                return(ContainerExecutor.ExitCode.Terminated.GetExitCode());
            }
            Shell.ShellCommandExecutor shExec = null;
            try
            {
                lfs.SetPermission(launchDst, ContainerExecutor.TaskLaunchScriptPermission);
                lfs.SetPermission(sb.GetWrapperScriptPath(), ContainerExecutor.TaskLaunchScriptPermission
                                  );
                // Setup command to run
                string[] command = GetRunCommand(sb.GetWrapperScriptPath().ToString(), containerIdStr
                                                 , userName, pidFile, this.GetConf());
                if (Log.IsDebugEnabled())
                {
                    Log.Debug("launchContainer: " + commandStr + " " + Joiner.On(" ").Join(command));
                }
                shExec = new Shell.ShellCommandExecutor(command, new FilePath(containerWorkDir.ToUri
                                                                                  ().GetPath()), container.GetLaunchContext().GetEnvironment());
                // sanitized env
                if (IsContainerActive(containerId))
                {
                    shExec.Execute();
                }
                else
                {
                    Log.Info("Container " + containerIdStr + " was marked as inactive. Returning terminated error"
                             );
                    return(ContainerExecutor.ExitCode.Terminated.GetExitCode());
                }
            }
            catch (IOException e)
            {
                if (null == shExec)
                {
                    return(-1);
                }
                int exitCode = shExec.GetExitCode();
                Log.Warn("Exit code from container " + containerId + " is : " + exitCode);
                // 143 (SIGTERM) and 137 (SIGKILL) exit codes means the container was
                // terminated/killed forcefully. In all other cases, log the
                // container-executor's output
                if (exitCode != ContainerExecutor.ExitCode.ForceKilled.GetExitCode() && exitCode
                    != ContainerExecutor.ExitCode.Terminated.GetExitCode())
                {
                    Log.Warn("Exception from container-launch with container ID: " + containerId + " and exit code: "
                             + exitCode, e);
                    LogOutput(shExec.GetOutput());
                    string diagnostics = "Exception from container-launch: \n" + StringUtils.StringifyException
                                             (e) + "\n" + shExec.GetOutput();
                    container.Handle(new ContainerDiagnosticsUpdateEvent(containerId, diagnostics));
                }
                else
                {
                    container.Handle(new ContainerDiagnosticsUpdateEvent(containerId, "Container killed on request. Exit code is "
                                                                         + exitCode));
                }
                return(exitCode);
            }
            finally
            {
                if (shExec != null)
                {
                    shExec.Close();
                }
            }
            return(0);
        }