示例#1
0
        public virtual int DumpAContainersLogs(string appId, string containerId, string nodeId
                                               , string jobOwner)
        {
            Path remoteRootLogDir = new Path(GetConf().Get(YarnConfiguration.NmRemoteAppLogDir
                                                           , YarnConfiguration.DefaultNmRemoteAppLogDir));
            string suffix          = LogAggregationUtils.GetRemoteNodeLogDirSuffix(GetConf());
            Path   remoteAppLogDir = LogAggregationUtils.GetRemoteAppLogDir(remoteRootLogDir, ConverterUtils
                                                                            .ToApplicationId(appId), jobOwner, suffix);
            RemoteIterator <FileStatus> nodeFiles;

            try
            {
                Path qualifiedLogDir = FileContext.GetFileContext(GetConf()).MakeQualified(remoteAppLogDir
                                                                                           );
                nodeFiles = FileContext.GetFileContext(qualifiedLogDir.ToUri(), GetConf()).ListStatus
                                (remoteAppLogDir);
            }
            catch (FileNotFoundException)
            {
                LogDirNotExist(remoteAppLogDir.ToString());
                return(-1);
            }
            bool foundContainerLogs = false;

            while (nodeFiles.HasNext())
            {
                FileStatus thisNodeFile = nodeFiles.Next();
                string     fileName     = thisNodeFile.GetPath().GetName();
                if (fileName.Contains(LogAggregationUtils.GetNodeString(nodeId)) && !fileName.EndsWith
                        (LogAggregationUtils.TmpFileSuffix))
                {
                    AggregatedLogFormat.LogReader reader = null;
                    try
                    {
                        reader = new AggregatedLogFormat.LogReader(GetConf(), thisNodeFile.GetPath());
                        if (DumpAContainerLogs(containerId, reader, System.Console.Out, thisNodeFile.GetModificationTime
                                                   ()) > -1)
                        {
                            foundContainerLogs = true;
                        }
                    }
                    finally
                    {
                        if (reader != null)
                        {
                            reader.Close();
                        }
                    }
                }
            }
            if (!foundContainerLogs)
            {
                ContainerLogNotFound(containerId);
                return(-1);
            }
            return(0);
        }
 public LogDeletionTask(Configuration conf, long retentionSecs, ApplicationClientProtocol
                        rmClient)
 {
     this.conf             = conf;
     this.retentionMillis  = retentionSecs * 1000;
     this.suffix           = LogAggregationUtils.GetRemoteNodeLogDirSuffix(conf);
     this.remoteRootLogDir = new Path(conf.Get(YarnConfiguration.NmRemoteAppLogDir, YarnConfiguration
                                               .DefaultNmRemoteAppLogDir));
     this.rmClient = rmClient;
 }
示例#3
0
        public virtual int DumpAllContainersLogs(ApplicationId appId, string appOwner, TextWriter
                                                 @out)
        {
            Path remoteRootLogDir = new Path(GetConf().Get(YarnConfiguration.NmRemoteAppLogDir
                                                           , YarnConfiguration.DefaultNmRemoteAppLogDir));
            string user         = appOwner;
            string logDirSuffix = LogAggregationUtils.GetRemoteNodeLogDirSuffix(GetConf());
            // TODO Change this to get a list of files from the LAS.
            Path remoteAppLogDir = LogAggregationUtils.GetRemoteAppLogDir(remoteRootLogDir, appId
                                                                          , user, logDirSuffix);
            RemoteIterator <FileStatus> nodeFiles;

            try
            {
                Path qualifiedLogDir = FileContext.GetFileContext(GetConf()).MakeQualified(remoteAppLogDir
                                                                                           );
                nodeFiles = FileContext.GetFileContext(qualifiedLogDir.ToUri(), GetConf()).ListStatus
                                (remoteAppLogDir);
            }
            catch (FileNotFoundException)
            {
                LogDirNotExist(remoteAppLogDir.ToString());
                return(-1);
            }
            bool foundAnyLogs = false;

            while (nodeFiles.HasNext())
            {
                FileStatus thisNodeFile = nodeFiles.Next();
                if (!thisNodeFile.GetPath().GetName().EndsWith(LogAggregationUtils.TmpFileSuffix))
                {
                    AggregatedLogFormat.LogReader reader = new AggregatedLogFormat.LogReader(GetConf(
                                                                                                 ), thisNodeFile.GetPath());
                    try
                    {
                        DataInputStream            valueStream;
                        AggregatedLogFormat.LogKey key = new AggregatedLogFormat.LogKey();
                        valueStream = reader.Next(key);
                        while (valueStream != null)
                        {
                            string containerString = "\n\nContainer: " + key + " on " + thisNodeFile.GetPath(
                                ).GetName();
                            @out.WriteLine(containerString);
                            @out.WriteLine(StringUtils.Repeat("=", containerString.Length));
                            while (true)
                            {
                                try
                                {
                                    AggregatedLogFormat.LogReader.ReadAContainerLogsForALogType(valueStream, @out, thisNodeFile
                                                                                                .GetModificationTime());
                                    foundAnyLogs = true;
                                }
                                catch (EOFException)
                                {
                                    break;
                                }
                            }
                            // Next container
                            key         = new AggregatedLogFormat.LogKey();
                            valueStream = reader.Next(key);
                        }
                    }
                    finally
                    {
                        reader.Close();
                    }
                }
            }
            if (!foundAnyLogs)
            {
                EmptyLogDir(remoteAppLogDir.ToString());
                return(-1);
            }
            return(0);
        }