Exemplo n.º 1
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public virtual bool StatusUpdate(TaskAttemptID taskAttemptID, TaskStatus taskStatus
                                         )
        {
            TaskAttemptId yarnAttemptID = TypeConverter.ToYarn(taskAttemptID);

            taskHeartbeatHandler.Progressing(yarnAttemptID);
            TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
                                                                                   ();
            taskAttemptStatus.id = yarnAttemptID;
            // Task sends the updated progress to the TT.
            taskAttemptStatus.progress = taskStatus.GetProgress();
            Log.Info("Progress of TaskAttempt " + taskAttemptID + " is : " + taskStatus.GetProgress
                         ());
            // Task sends the updated state-string to the TT.
            taskAttemptStatus.stateString = taskStatus.GetStateString();
            // Task sends the updated phase to the TT.
            taskAttemptStatus.phase = TypeConverter.ToYarn(taskStatus.GetPhase());
            // Counters are updated by the task. Convert counters into new format as
            // that is the primary storage format inside the AM to avoid multiple
            // conversions and unnecessary heap usage.
            taskAttemptStatus.counters = new Counters(taskStatus.GetCounters());
            // Map Finish time set by the task (map only)
            if (taskStatus.GetIsMap() && taskStatus.GetMapFinishTime() != 0)
            {
                taskAttemptStatus.mapFinishTime = taskStatus.GetMapFinishTime();
            }
            // Shuffle Finish time set by the task (reduce only).
            if (!taskStatus.GetIsMap() && taskStatus.GetShuffleFinishTime() != 0)
            {
                taskAttemptStatus.shuffleFinishTime = taskStatus.GetShuffleFinishTime();
            }
            // Sort finish time set by the task (reduce only).
            if (!taskStatus.GetIsMap() && taskStatus.GetSortFinishTime() != 0)
            {
                taskAttemptStatus.sortFinishTime = taskStatus.GetSortFinishTime();
            }
            // Not Setting the task state. Used by speculation - will be set in TaskAttemptImpl
            //taskAttemptStatus.taskState =  TypeConverter.toYarn(taskStatus.getRunState());
            //set the fetch failures
            if (taskStatus.GetFetchFailedMaps() != null && taskStatus.GetFetchFailedMaps().Count
                > 0)
            {
                taskAttemptStatus.fetchFailedMaps = new AList <TaskAttemptId>();
                foreach (TaskAttemptID failedMapId in taskStatus.GetFetchFailedMaps())
                {
                    taskAttemptStatus.fetchFailedMaps.AddItem(TypeConverter.ToYarn(failedMapId));
                }
            }
            // Task sends the information about the nextRecordRange to the TT
            //    TODO: The following are not needed here, but needed to be set somewhere inside AppMaster.
            //    taskStatus.getRunState(); // Set by the TT/JT. Transform into a state TODO
            //    taskStatus.getStartTime(); // Used to be set by the TaskTracker. This should be set by getTask().
            //    taskStatus.getFinishTime(); // Used to be set by TT/JT. Should be set when task finishes
            //    // This was used by TT to do counter updates only once every minute. So this
            //    // isn't ever changed by the Task itself.
            //    taskStatus.getIncludeCounters();
            context.GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(taskAttemptStatus
                                                                              .id, taskAttemptStatus));
            return(true);
        }
Exemplo n.º 2
0
 public virtual void CompleteMap(TaskAttemptID taskAttemptID)
 {
     lock (this)
     {
         ++numMapTasksCompleted;
     }
 }
Exemplo n.º 3
0
 public MapTaskStatus(TaskAttemptID taskid, float progress, int numSlots, TaskStatus.State
                      runState, string diagnosticInfo, string stateString, string taskTracker, TaskStatus.Phase
                      phase, Counters counters)
     : base(taskid, progress, numSlots, runState, diagnosticInfo, stateString, taskTracker
            , phase, counters)
 {
 }
Exemplo n.º 4
0
 /// <exception cref="System.IO.IOException"/>
 public override void FatalError(TaskAttemptID taskId, string msg)
 {
     lock (this)
     {
         LocalJobRunner.Log.Fatal("Fatal: " + msg + "from task: " + taskId);
     }
 }
Exemplo n.º 5
0
 public virtual void CompleteReduce(TaskAttemptID taskAttemptID)
 {
     lock (this)
     {
         ++numReduceTasksCompleted;
     }
 }
Exemplo n.º 6
0
        internal static FilePath GetAttemptDir(TaskAttemptID taskid, bool isCleanup)
        {
            string cleanupSuffix = isCleanup ? ".cleanup" : string.Empty;

            return(new FilePath(GetJobDir(((JobID)taskid.GetJobID())), taskid + cleanupSuffix
                                ));
        }
Exemplo n.º 7
0
 /// <exception cref="System.IO.IOException"/>
 public override void FsError(TaskAttemptID taskId, string message)
 {
     lock (this)
     {
         LocalJobRunner.Log.Fatal("FSError: " + message + "from task: " + taskId);
     }
 }
Exemplo n.º 8
0
 /// <exception cref="System.IO.IOException"/>
 public static void SyncLogs(string logLocation, TaskAttemptID taskid, bool isCleanup
                             )
 {
     lock (typeof(TaskLog))
     {
         System.Console.Out.Flush();
         System.Console.Error.Flush();
         Enumeration <Logger> allLoggers = LogManager.GetCurrentLoggers();
         while (allLoggers.MoveNext())
         {
             Logger l = allLoggers.Current;
             Enumeration <Appender> allAppenders = l.GetAllAppenders();
             while (allAppenders.MoveNext())
             {
                 Appender a = allAppenders.Current;
                 if (a is TaskLogAppender)
                 {
                     ((TaskLogAppender)a).Flush();
                 }
             }
         }
         if (currentTaskid != taskid)
         {
             currentTaskid = taskid;
             ResetPrevLengths(logLocation);
         }
         WriteToIndexFile(logLocation, isCleanup);
     }
 }
Exemplo n.º 9
0
        /// <exception cref="System.IO.IOException"/>
        private static TaskLog.LogFileDetail GetLogFileDetail(TaskAttemptID taskid, TaskLog.LogName
                                                              filter, bool isCleanup)
        {
            FilePath       indexFile = GetIndexFile(taskid, isCleanup);
            BufferedReader fis       = new BufferedReader(new InputStreamReader(SecureIOUtils.OpenForRead
                                                                                    (indexFile, ObtainLogDirOwner(taskid), null), Charsets.Utf8));

            //the format of the index file is
            //LOG_DIR: <the dir where the task logs are really stored>
            //stdout:<start-offset in the stdout file> <length>
            //stderr:<start-offset in the stderr file> <length>
            //syslog:<start-offset in the syslog file> <length>
            TaskLog.LogFileDetail l = new TaskLog.LogFileDetail();
            string str = null;

            try
            {
                str = fis.ReadLine();
                if (str == null)
                {
                    // the file doesn't have anything
                    throw new IOException("Index file for the log of " + taskid + " doesn't exist.");
                }
                l.location = Sharpen.Runtime.Substring(str, str.IndexOf(TaskLog.LogFileDetail.Location
                                                                        ) + TaskLog.LogFileDetail.Location.Length);
                // special cases are the debugout and profile.out files. They are
                // guaranteed
                // to be associated with each task attempt since jvm reuse is disabled
                // when profiling/debugging is enabled
                if (filter.Equals(TaskLog.LogName.Debugout) || filter.Equals(TaskLog.LogName.Profile
                                                                             ))
                {
                    l.length = new FilePath(l.location, filter.ToString()).Length();
                    l.start  = 0;
                    fis.Close();
                    return(l);
                }
                str = fis.ReadLine();
                while (str != null)
                {
                    // look for the exact line containing the logname
                    if (str.Contains(filter.ToString()))
                    {
                        str = Sharpen.Runtime.Substring(str, filter.ToString().Length + 1);
                        string[] startAndLen = str.Split(" ");
                        l.start  = long.Parse(startAndLen[0]);
                        l.length = long.Parse(startAndLen[1]);
                        break;
                    }
                    str = fis.ReadLine();
                }
                fis.Close();
                fis = null;
            }
            finally
            {
                IOUtils.Cleanup(Log, fis);
            }
            return(l);
        }
Exemplo n.º 10
0
        /// <exception cref="System.IO.IOException"/>
        public virtual string ReadTaskLog(TaskLog.LogName filter, TaskAttemptID taskId, bool
                                          isCleanup)
        {
            // string buffer to store task log
            StringBuilder result = new StringBuilder();
            int           res;
            // reads the whole tasklog into inputstream
            InputStream taskLogReader = new TaskLog.Reader(taskId, filter, 0, -1, isCleanup);

            // construct string log from inputstream.
            byte[] b = new byte[65536];
            while (true)
            {
                res = taskLogReader.Read(b);
                if (res > 0)
                {
                    result.Append(Sharpen.Runtime.GetStringForBytes(b));
                }
                else
                {
                    break;
                }
            }
            taskLogReader.Close();
            // trim the string and return it
            string str = result.ToString();

            str = str.Trim();
            return(str);
        }
Exemplo n.º 11
0
 public Context(TaskAttemptID reduceId, JobConf jobConf, FileSystem localFS, TaskUmbilicalProtocol
                umbilical, LocalDirAllocator localDirAllocator, Reporter reporter, CompressionCodec
                codec, Type combinerClass, Task.CombineOutputCollector <K, V> combineCollector,
                Counters.Counter spilledRecordsCounter, Counters.Counter reduceCombineInputCounter
                , Counters.Counter shuffledMapsCounter, Counters.Counter reduceShuffleBytes, Counters.Counter
                failedShuffleCounter, Counters.Counter mergedMapOutputsCounter, TaskStatus status
                , Progress copyPhase, Progress mergePhase, Task reduceTask, MapOutputFile mapOutputFile
                , IDictionary <TaskAttemptID, MapOutputFile> localMapFiles)
 {
     this.reduceId                  = reduceId;
     this.jobConf                   = jobConf;
     this.localFS                   = localFS;
     this.umbilical                 = umbilical;
     this.localDirAllocator         = localDirAllocator;
     this.reporter                  = reporter;
     this.codec                     = codec;
     this.combinerClass             = combinerClass;
     this.combineCollector          = combineCollector;
     this.spilledRecordsCounter     = spilledRecordsCounter;
     this.reduceCombineInputCounter = reduceCombineInputCounter;
     this.shuffledMapsCounter       = shuffledMapsCounter;
     this.reduceShuffleBytes        = reduceShuffleBytes;
     this.failedShuffleCounter      = failedShuffleCounter;
     this.mergedMapOutputsCounter   = mergedMapOutputsCounter;
     this.status                    = status;
     this.copyPhase                 = copyPhase;
     this.mergePhase                = mergePhase;
     this.reduceTask                = reduceTask;
     this.mapOutputFile             = mapOutputFile;
     this.localMapFiles             = localMapFiles;
 }
Exemplo n.º 12
0
 /// <exception cref="System.IO.IOException"/>
 public virtual bool Ping(TaskAttemptID taskAttemptID)
 {
     if (Log.IsDebugEnabled())
     {
         Log.Debug("Ping from " + taskAttemptID.ToString());
     }
     return(true);
 }
Exemplo n.º 13
0
 public _TaskStatus_187(int maxSize, TaskAttemptID baseArg1, float baseArg2, int baseArg3
                        , TaskStatus.State baseArg4, string baseArg5, string baseArg6, string baseArg7,
                        TaskStatus.Phase baseArg8, Counters baseArg9)
     : base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7, baseArg8
            , baseArg9)
 {
     this.maxSize = maxSize;
 }
Exemplo n.º 14
0
 public virtual void LaunchMap(TaskAttemptID taskAttemptID)
 {
     lock (this)
     {
         ++numMapTasksLaunched;
         DecWaitingMaps(((JobID)taskAttemptID.GetJobID()), 1);
     }
 }
Exemplo n.º 15
0
 public virtual void LaunchReduce(TaskAttemptID taskAttemptID)
 {
     lock (this)
     {
         ++numReduceTasksLaunched;
         DecWaitingReduces(((JobID)taskAttemptID.GetJobID()), 1);
     }
 }
Exemplo n.º 16
0
 /// <exception cref="System.IO.IOException"/>
 public virtual void ReportNextRecordRange(TaskAttemptID taskAttemptID, SortedRanges.Range
                                           range)
 {
     // This is used when the feature of skipping records is enabled.
     // This call exists as a hadoop mapreduce legacy wherein all changes in
     // counters/progress/phase/output-size are reported through statusUpdate()
     // call but not the next record range information.
     throw new IOException("Not yet implemented.");
 }
Exemplo n.º 17
0
 internal static Org.Apache.Hadoop.Mapred.TaskStatus CreateTaskStatus(bool isMap,
                                                                      TaskAttemptID taskId, float progress, int numSlots, TaskStatus.State runState, string
                                                                      diagnosticInfo, string stateString, string taskTracker, TaskStatus.Phase phase,
                                                                      Counters counters)
 {
     return((isMap) ? new MapTaskStatus(taskId, progress, numSlots, runState, diagnosticInfo
                                        , stateString, taskTracker, phase, counters) : new ReduceTaskStatus(taskId, progress
                                                                                                            , numSlots, runState, diagnosticInfo, stateString, taskTracker, phase, counters));
 }
Exemplo n.º 18
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void Done(TaskAttemptID taskAttemptID)
        {
            Log.Info("Done acknowledgement from " + taskAttemptID.ToString());
            TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);

            taskHeartbeatHandler.Progressing(attemptID);
            context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType
                                                                  .TaDone));
        }
Exemplo n.º 19
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void FatalError(TaskAttemptID taskAttemptID, string msg)
        {
            // This happens only in Child and in the Task.
            Log.Fatal("Task: " + taskAttemptID + " - exited : " + msg);
            ReportDiagnosticInfo(taskAttemptID, "Error: " + msg);
            TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);

            context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType
                                                                  .TaFailmsg));
        }
Exemplo n.º 20
0
        //////////////////////////////////////////////////////////////////////////////
        // Factory-like methods to create/read/write appropriate TaskStatus objects
        //////////////////////////////////////////////////////////////////////////////
        /// <exception cref="System.IO.IOException"/>
        internal static Org.Apache.Hadoop.Mapred.TaskStatus CreateTaskStatus(DataInput @in
                                                                             , TaskAttemptID taskId, float progress, int numSlots, TaskStatus.State runState,
                                                                             string diagnosticInfo, string stateString, string taskTracker, TaskStatus.Phase
                                                                             phase, Counters counters)
        {
            bool isMap = @in.ReadBoolean();

            return(CreateTaskStatus(isMap, taskId, progress, numSlots, runState, diagnosticInfo
                                    , stateString, taskTracker, phase, counters));
        }
Exemplo n.º 21
0
        /// <summary>Obtain the owner of the log dir.</summary>
        /// <remarks>
        /// Obtain the owner of the log dir. This is
        /// determined by checking the job's log directory.
        /// </remarks>
        /// <exception cref="System.IO.IOException"/>
        internal static string ObtainLogDirOwner(TaskAttemptID taskid)
        {
            Configuration conf      = new Configuration();
            FileSystem    raw       = FileSystem.GetLocal(conf).GetRaw();
            Path          jobLogDir = new Path(GetJobDir(((JobID)taskid.GetJobID())).GetAbsolutePath()
                                               );
            FileStatus jobStat = raw.GetFileStatus(jobLogDir);

            return(jobStat.GetOwner());
        }
Exemplo n.º 22
0
        /// <summary>Get the running task attempt IDs for this task</summary>
        public virtual ICollection <TaskAttemptID> GetRunningTaskAttempts()
        {
            ICollection <TaskAttemptID> attempts = new AList <TaskAttemptID>();

            foreach (TaskAttemptID id in base.GetRunningTaskAttemptIds())
            {
                attempts.AddItem(TaskAttemptID.Downgrade(id));
            }
            return(attempts);
        }
Exemplo n.º 23
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void FsError(TaskAttemptID taskAttemptID, string message)
        {
            // This happens only in Child.
            Log.Fatal("Task: " + taskAttemptID + " - failed due to FSError: " + message);
            ReportDiagnosticInfo(taskAttemptID, "FSError: " + message);
            TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);

            context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType
                                                                  .TaFailmsg));
        }
Exemplo n.º 24
0
 public ReduceTask(string jobFile, TaskAttemptID taskId, int partition, int numMaps
                   , int numSlotsRequired)
     : base(jobFile, taskId, partition, numSlotsRequired)
 {
     mapOutputFilesOnDisk = new TreeSet <FileStatus>(mapOutputFileComparator);
     {
         GetProgress().SetStatus("reduce");
         SetPhase(TaskStatus.Phase.Shuffle);
     }
     // A sorted set for keeping a set of map output files on disk
     this.numMaps = numMaps;
 }
Exemplo n.º 25
0
 public static FilePath GetTaskLogFile(TaskAttemptID taskid, bool isCleanup, TaskLog.LogName
                                       filter)
 {
     if (GetMRv2LogDir() != null)
     {
         return(new FilePath(GetMRv2LogDir(), filter.ToString()));
     }
     else
     {
         return(new FilePath(GetAttemptDir(taskid, isCleanup), filter.ToString()));
     }
 }
Exemplo n.º 26
0
        /// <summary>test TaskAttemptID</summary>
        public virtual void TestTaskAttemptID()
        {
            TaskAttemptID task = new TaskAttemptID("001", 2, true, 3, 4);

            NUnit.Framework.Assert.AreEqual("attempt_001_0002_m_000003_4", TaskAttemptID.GetTaskAttemptIDsPattern
                                                ("001", 2, true, 3, 4));
            NUnit.Framework.Assert.AreEqual("task_001_0002_m_000003", ((TaskID)task.GetTaskID
                                                                           ()).ToString());
            NUnit.Framework.Assert.AreEqual("attempt_001_0001_r_000002_3", TaskAttemptID.GetTaskAttemptIDsPattern
                                                ("001", 1, TaskType.Reduce, 2, 3));
            NUnit.Framework.Assert.AreEqual("001_0001_m_000001_2", TaskAttemptID.GetTaskAttemptIDsPatternWOPrefix
                                                ("001", 1, TaskType.Map, 1, 2).ToString());
        }
Exemplo n.º 27
0
        /// <summary>
        /// TaskAttempt is reporting that it is in commit_pending and it is waiting for
        /// the commit Response
        /// <br />
        /// Commit it a two-phased protocol.
        /// </summary>
        /// <remarks>
        /// TaskAttempt is reporting that it is in commit_pending and it is waiting for
        /// the commit Response
        /// <br />
        /// Commit it a two-phased protocol. First the attempt informs the
        /// ApplicationMaster that it is
        /// <see cref="CommitPending(TaskAttemptID, TaskStatus)"/>
        /// . Then it repeatedly polls
        /// the ApplicationMaster whether it
        /// <see cref="CanCommit(TaskAttemptID)"/>
        /// This is
        /// a legacy from the centralized commit protocol handling by the JobTracker.
        /// </remarks>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        public virtual void CommitPending(TaskAttemptID taskAttemptID, TaskStatus taskStatsu
                                          )
        {
            Log.Info("Commit-pending state update from " + taskAttemptID.ToString());
            // An attempt is asking if it can commit its output. This can be decided
            // only by the task which is managing the multiple attempts. So redirect the
            // request there.
            TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);

            taskHeartbeatHandler.Progressing(attemptID);
            //Ignorable TaskStatus? - since a task will send a LastStatusUpdate
            context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType
                                                                  .TaCommitPending));
        }
Exemplo n.º 28
0
            /// <exception cref="System.IO.IOException"/>
            public override void Done(TaskAttemptID taskId)
            {
                int taskIndex = this.mapIds.IndexOf(taskId);

                if (taskIndex >= 0)
                {
                    // mapping
                    this.status.SetMapProgress(1.0f);
                }
                else
                {
                    this.status.SetReduceProgress(1.0f);
                }
            }
Exemplo n.º 29
0
        /// <summary>test without TASK_LOG_DIR</summary>
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestTaskLogWithoutTaskLogDir()
        {
            // TaskLog tasklog= new TaskLog();
            Runtime.ClearProperty(YarnConfiguration.YarnAppContainerLogDir);
            // test TaskLog
            NUnit.Framework.Assert.AreEqual(TaskLog.GetMRv2LogDir(), null);
            TaskAttemptID taid = Org.Mockito.Mockito.Mock <TaskAttemptID>();
            JobID         jid  = new JobID("job", 1);

            Org.Mockito.Mockito.When(((JobID)taid.GetJobID())).ThenReturn(jid);
            Org.Mockito.Mockito.When(taid.ToString()).ThenReturn("JobId");
            FilePath f = TaskLog.GetTaskLogFile(taid, true, TaskLog.LogName.Stdout);

            NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("stdout"));
        }
Exemplo n.º 30
0
 internal static FilePath GetRealTaskLogFileLocation(TaskAttemptID taskid, bool isCleanup
                                                     , TaskLog.LogName filter)
 {
     TaskLog.LogFileDetail l;
     try
     {
         l = GetLogFileDetail(taskid, filter, isCleanup);
     }
     catch (IOException ie)
     {
         Log.Error("getTaskLogFileDetail threw an exception " + ie);
         return(null);
     }
     return(new FilePath(l.location, filter.ToString()));
 }