/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual bool StatusUpdate(TaskAttemptID taskAttemptID, TaskStatus taskStatus
)
{
TaskAttemptId yarnAttemptID = TypeConverter.ToYarn(taskAttemptID);
taskHeartbeatHandler.Progressing(yarnAttemptID);
TaskAttemptStatusUpdateEvent.TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus
();
taskAttemptStatus.id = yarnAttemptID;
// Task sends the updated progress to the TT.
taskAttemptStatus.progress = taskStatus.GetProgress();
Log.Info("Progress of TaskAttempt " + taskAttemptID + " is : " + taskStatus.GetProgress
());
// Task sends the updated state-string to the TT.
taskAttemptStatus.stateString = taskStatus.GetStateString();
// Task sends the updated phase to the TT.
taskAttemptStatus.phase = TypeConverter.ToYarn(taskStatus.GetPhase());
// Counters are updated by the task. Convert counters into new format as
// that is the primary storage format inside the AM to avoid multiple
// conversions and unnecessary heap usage.
taskAttemptStatus.counters = new Counters(taskStatus.GetCounters());
// Map Finish time set by the task (map only)
if (taskStatus.GetIsMap() && taskStatus.GetMapFinishTime() != 0)
{
taskAttemptStatus.mapFinishTime = taskStatus.GetMapFinishTime();
}
// Shuffle Finish time set by the task (reduce only).
if (!taskStatus.GetIsMap() && taskStatus.GetShuffleFinishTime() != 0)
{
taskAttemptStatus.shuffleFinishTime = taskStatus.GetShuffleFinishTime();
}
// Sort finish time set by the task (reduce only).
if (!taskStatus.GetIsMap() && taskStatus.GetSortFinishTime() != 0)
{
taskAttemptStatus.sortFinishTime = taskStatus.GetSortFinishTime();
}
// Not Setting the task state. Used by speculation - will be set in TaskAttemptImpl
//taskAttemptStatus.taskState = TypeConverter.toYarn(taskStatus.getRunState());
//set the fetch failures
if (taskStatus.GetFetchFailedMaps() != null && taskStatus.GetFetchFailedMaps().Count
> 0)
{
taskAttemptStatus.fetchFailedMaps = new AList <TaskAttemptId>();
foreach (TaskAttemptID failedMapId in taskStatus.GetFetchFailedMaps())
{
taskAttemptStatus.fetchFailedMaps.AddItem(TypeConverter.ToYarn(failedMapId));
}
}
// Task sends the information about the nextRecordRange to the TT
// TODO: The following are not needed here, but needed to be set somewhere inside AppMaster.
// taskStatus.getRunState(); // Set by the TT/JT. Transform into a state TODO
// taskStatus.getStartTime(); // Used to be set by the TaskTracker. This should be set by getTask().
// taskStatus.getFinishTime(); // Used to be set by TT/JT. Should be set when task finishes
// // This was used by TT to do counter updates only once every minute. So this
// // isn't ever changed by the Task itself.
// taskStatus.getIncludeCounters();
context.GetEventHandler().Handle(new TaskAttemptStatusUpdateEvent(taskAttemptStatus
.id, taskAttemptStatus));
return(true);
}
public virtual void CompleteMap(TaskAttemptID taskAttemptID)
{
lock (this)
{
++numMapTasksCompleted;
}
}
public MapTaskStatus(TaskAttemptID taskid, float progress, int numSlots, TaskStatus.State
runState, string diagnosticInfo, string stateString, string taskTracker, TaskStatus.Phase
phase, Counters counters)
: base(taskid, progress, numSlots, runState, diagnosticInfo, stateString, taskTracker
, phase, counters)
{
}
/// <exception cref="System.IO.IOException"/>
public override void FatalError(TaskAttemptID taskId, string msg)
{
lock (this)
{
LocalJobRunner.Log.Fatal("Fatal: " + msg + "from task: " + taskId);
}
}
public virtual void CompleteReduce(TaskAttemptID taskAttemptID)
{
lock (this)
{
++numReduceTasksCompleted;
}
}
internal static FilePath GetAttemptDir(TaskAttemptID taskid, bool isCleanup)
{
string cleanupSuffix = isCleanup ? ".cleanup" : string.Empty;
return(new FilePath(GetJobDir(((JobID)taskid.GetJobID())), taskid + cleanupSuffix
));
}
/// <exception cref="System.IO.IOException"/>
public override void FsError(TaskAttemptID taskId, string message)
{
lock (this)
{
LocalJobRunner.Log.Fatal("FSError: " + message + "from task: " + taskId);
}
}
/// <exception cref="System.IO.IOException"/>
public static void SyncLogs(string logLocation, TaskAttemptID taskid, bool isCleanup
)
{
lock (typeof(TaskLog))
{
System.Console.Out.Flush();
System.Console.Error.Flush();
Enumeration <Logger> allLoggers = LogManager.GetCurrentLoggers();
while (allLoggers.MoveNext())
{
Logger l = allLoggers.Current;
Enumeration <Appender> allAppenders = l.GetAllAppenders();
while (allAppenders.MoveNext())
{
Appender a = allAppenders.Current;
if (a is TaskLogAppender)
{
((TaskLogAppender)a).Flush();
}
}
}
if (currentTaskid != taskid)
{
currentTaskid = taskid;
ResetPrevLengths(logLocation);
}
WriteToIndexFile(logLocation, isCleanup);
}
}
/// <exception cref="System.IO.IOException"/>
private static TaskLog.LogFileDetail GetLogFileDetail(TaskAttemptID taskid, TaskLog.LogName
filter, bool isCleanup)
{
FilePath indexFile = GetIndexFile(taskid, isCleanup);
BufferedReader fis = new BufferedReader(new InputStreamReader(SecureIOUtils.OpenForRead
(indexFile, ObtainLogDirOwner(taskid), null), Charsets.Utf8));
//the format of the index file is
//LOG_DIR: <the dir where the task logs are really stored>
//stdout:<start-offset in the stdout file> <length>
//stderr:<start-offset in the stderr file> <length>
//syslog:<start-offset in the syslog file> <length>
TaskLog.LogFileDetail l = new TaskLog.LogFileDetail();
string str = null;
try
{
str = fis.ReadLine();
if (str == null)
{
// the file doesn't have anything
throw new IOException("Index file for the log of " + taskid + " doesn't exist.");
}
l.location = Sharpen.Runtime.Substring(str, str.IndexOf(TaskLog.LogFileDetail.Location
) + TaskLog.LogFileDetail.Location.Length);
// special cases are the debugout and profile.out files. They are
// guaranteed
// to be associated with each task attempt since jvm reuse is disabled
// when profiling/debugging is enabled
if (filter.Equals(TaskLog.LogName.Debugout) || filter.Equals(TaskLog.LogName.Profile
))
{
l.length = new FilePath(l.location, filter.ToString()).Length();
l.start = 0;
fis.Close();
return(l);
}
str = fis.ReadLine();
while (str != null)
{
// look for the exact line containing the logname
if (str.Contains(filter.ToString()))
{
str = Sharpen.Runtime.Substring(str, filter.ToString().Length + 1);
string[] startAndLen = str.Split(" ");
l.start = long.Parse(startAndLen[0]);
l.length = long.Parse(startAndLen[1]);
break;
}
str = fis.ReadLine();
}
fis.Close();
fis = null;
}
finally
{
IOUtils.Cleanup(Log, fis);
}
return(l);
}
/// <exception cref="System.IO.IOException"/>
public virtual string ReadTaskLog(TaskLog.LogName filter, TaskAttemptID taskId, bool
isCleanup)
{
// string buffer to store task log
StringBuilder result = new StringBuilder();
int res;
// reads the whole tasklog into inputstream
InputStream taskLogReader = new TaskLog.Reader(taskId, filter, 0, -1, isCleanup);
// construct string log from inputstream.
byte[] b = new byte[65536];
while (true)
{
res = taskLogReader.Read(b);
if (res > 0)
{
result.Append(Sharpen.Runtime.GetStringForBytes(b));
}
else
{
break;
}
}
taskLogReader.Close();
// trim the string and return it
string str = result.ToString();
str = str.Trim();
return(str);
}
public Context(TaskAttemptID reduceId, JobConf jobConf, FileSystem localFS, TaskUmbilicalProtocol
umbilical, LocalDirAllocator localDirAllocator, Reporter reporter, CompressionCodec
codec, Type combinerClass, Task.CombineOutputCollector <K, V> combineCollector,
Counters.Counter spilledRecordsCounter, Counters.Counter reduceCombineInputCounter
, Counters.Counter shuffledMapsCounter, Counters.Counter reduceShuffleBytes, Counters.Counter
failedShuffleCounter, Counters.Counter mergedMapOutputsCounter, TaskStatus status
, Progress copyPhase, Progress mergePhase, Task reduceTask, MapOutputFile mapOutputFile
, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles)
{
this.reduceId = reduceId;
this.jobConf = jobConf;
this.localFS = localFS;
this.umbilical = umbilical;
this.localDirAllocator = localDirAllocator;
this.reporter = reporter;
this.codec = codec;
this.combinerClass = combinerClass;
this.combineCollector = combineCollector;
this.spilledRecordsCounter = spilledRecordsCounter;
this.reduceCombineInputCounter = reduceCombineInputCounter;
this.shuffledMapsCounter = shuffledMapsCounter;
this.reduceShuffleBytes = reduceShuffleBytes;
this.failedShuffleCounter = failedShuffleCounter;
this.mergedMapOutputsCounter = mergedMapOutputsCounter;
this.status = status;
this.copyPhase = copyPhase;
this.mergePhase = mergePhase;
this.reduceTask = reduceTask;
this.mapOutputFile = mapOutputFile;
this.localMapFiles = localMapFiles;
}
/// <exception cref="System.IO.IOException"/>
public virtual bool Ping(TaskAttemptID taskAttemptID)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Ping from " + taskAttemptID.ToString());
}
return(true);
}
public _TaskStatus_187(int maxSize, TaskAttemptID baseArg1, float baseArg2, int baseArg3
, TaskStatus.State baseArg4, string baseArg5, string baseArg6, string baseArg7,
TaskStatus.Phase baseArg8, Counters baseArg9)
: base(baseArg1, baseArg2, baseArg3, baseArg4, baseArg5, baseArg6, baseArg7, baseArg8
, baseArg9)
{
this.maxSize = maxSize;
}
public virtual void LaunchMap(TaskAttemptID taskAttemptID)
{
lock (this)
{
++numMapTasksLaunched;
DecWaitingMaps(((JobID)taskAttemptID.GetJobID()), 1);
}
}
public virtual void LaunchReduce(TaskAttemptID taskAttemptID)
{
lock (this)
{
++numReduceTasksLaunched;
DecWaitingReduces(((JobID)taskAttemptID.GetJobID()), 1);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReportNextRecordRange(TaskAttemptID taskAttemptID, SortedRanges.Range
range)
{
// This is used when the feature of skipping records is enabled.
// This call exists as a hadoop mapreduce legacy wherein all changes in
// counters/progress/phase/output-size are reported through statusUpdate()
// call but not the next record range information.
throw new IOException("Not yet implemented.");
}
internal static Org.Apache.Hadoop.Mapred.TaskStatus CreateTaskStatus(bool isMap,
TaskAttemptID taskId, float progress, int numSlots, TaskStatus.State runState, string
diagnosticInfo, string stateString, string taskTracker, TaskStatus.Phase phase,
Counters counters)
{
return((isMap) ? new MapTaskStatus(taskId, progress, numSlots, runState, diagnosticInfo
, stateString, taskTracker, phase, counters) : new ReduceTaskStatus(taskId, progress
, numSlots, runState, diagnosticInfo, stateString, taskTracker, phase, counters));
}
/// <exception cref="System.IO.IOException"/>
public virtual void Done(TaskAttemptID taskAttemptID)
{
Log.Info("Done acknowledgement from " + taskAttemptID.ToString());
TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);
taskHeartbeatHandler.Progressing(attemptID);
context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType
.TaDone));
}
/// <exception cref="System.IO.IOException"/>
public virtual void FatalError(TaskAttemptID taskAttemptID, string msg)
{
// This happens only in Child and in the Task.
Log.Fatal("Task: " + taskAttemptID + " - exited : " + msg);
ReportDiagnosticInfo(taskAttemptID, "Error: " + msg);
TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);
context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType
.TaFailmsg));
}
//////////////////////////////////////////////////////////////////////////////
// Factory-like methods to create/read/write appropriate TaskStatus objects
//////////////////////////////////////////////////////////////////////////////
/// <exception cref="System.IO.IOException"/>
internal static Org.Apache.Hadoop.Mapred.TaskStatus CreateTaskStatus(DataInput @in
, TaskAttemptID taskId, float progress, int numSlots, TaskStatus.State runState,
string diagnosticInfo, string stateString, string taskTracker, TaskStatus.Phase
phase, Counters counters)
{
bool isMap = @in.ReadBoolean();
return(CreateTaskStatus(isMap, taskId, progress, numSlots, runState, diagnosticInfo
, stateString, taskTracker, phase, counters));
}
/// <summary>Obtain the owner of the log dir.</summary>
/// <remarks>
/// Obtain the owner of the log dir. This is
/// determined by checking the job's log directory.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
internal static string ObtainLogDirOwner(TaskAttemptID taskid)
{
Configuration conf = new Configuration();
FileSystem raw = FileSystem.GetLocal(conf).GetRaw();
Path jobLogDir = new Path(GetJobDir(((JobID)taskid.GetJobID())).GetAbsolutePath()
);
FileStatus jobStat = raw.GetFileStatus(jobLogDir);
return(jobStat.GetOwner());
}
/// <summary>Get the running task attempt IDs for this task</summary>
public virtual ICollection <TaskAttemptID> GetRunningTaskAttempts()
{
ICollection <TaskAttemptID> attempts = new AList <TaskAttemptID>();
foreach (TaskAttemptID id in base.GetRunningTaskAttemptIds())
{
attempts.AddItem(TaskAttemptID.Downgrade(id));
}
return(attempts);
}
/// <exception cref="System.IO.IOException"/>
public virtual void FsError(TaskAttemptID taskAttemptID, string message)
{
// This happens only in Child.
Log.Fatal("Task: " + taskAttemptID + " - failed due to FSError: " + message);
ReportDiagnosticInfo(taskAttemptID, "FSError: " + message);
TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);
context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType
.TaFailmsg));
}
public ReduceTask(string jobFile, TaskAttemptID taskId, int partition, int numMaps
, int numSlotsRequired)
: base(jobFile, taskId, partition, numSlotsRequired)
{
mapOutputFilesOnDisk = new TreeSet <FileStatus>(mapOutputFileComparator);
{
GetProgress().SetStatus("reduce");
SetPhase(TaskStatus.Phase.Shuffle);
}
// A sorted set for keeping a set of map output files on disk
this.numMaps = numMaps;
}
public static FilePath GetTaskLogFile(TaskAttemptID taskid, bool isCleanup, TaskLog.LogName
filter)
{
if (GetMRv2LogDir() != null)
{
return(new FilePath(GetMRv2LogDir(), filter.ToString()));
}
else
{
return(new FilePath(GetAttemptDir(taskid, isCleanup), filter.ToString()));
}
}
/// <summary>test TaskAttemptID</summary>
public virtual void TestTaskAttemptID()
{
TaskAttemptID task = new TaskAttemptID("001", 2, true, 3, 4);
NUnit.Framework.Assert.AreEqual("attempt_001_0002_m_000003_4", TaskAttemptID.GetTaskAttemptIDsPattern
("001", 2, true, 3, 4));
NUnit.Framework.Assert.AreEqual("task_001_0002_m_000003", ((TaskID)task.GetTaskID
()).ToString());
NUnit.Framework.Assert.AreEqual("attempt_001_0001_r_000002_3", TaskAttemptID.GetTaskAttemptIDsPattern
("001", 1, TaskType.Reduce, 2, 3));
NUnit.Framework.Assert.AreEqual("001_0001_m_000001_2", TaskAttemptID.GetTaskAttemptIDsPatternWOPrefix
("001", 1, TaskType.Map, 1, 2).ToString());
}
/// <summary>
/// TaskAttempt is reporting that it is in commit_pending and it is waiting for
/// the commit Response
/// <br />
/// Commit it a two-phased protocol.
/// </summary>
/// <remarks>
/// TaskAttempt is reporting that it is in commit_pending and it is waiting for
/// the commit Response
/// <br />
/// Commit it a two-phased protocol. First the attempt informs the
/// ApplicationMaster that it is
/// <see cref="CommitPending(TaskAttemptID, TaskStatus)"/>
/// . Then it repeatedly polls
/// the ApplicationMaster whether it
/// <see cref="CanCommit(TaskAttemptID)"/>
/// This is
/// a legacy from the centralized commit protocol handling by the JobTracker.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void CommitPending(TaskAttemptID taskAttemptID, TaskStatus taskStatsu
)
{
Log.Info("Commit-pending state update from " + taskAttemptID.ToString());
// An attempt is asking if it can commit its output. This can be decided
// only by the task which is managing the multiple attempts. So redirect the
// request there.
TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);
taskHeartbeatHandler.Progressing(attemptID);
//Ignorable TaskStatus? - since a task will send a LastStatusUpdate
context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType
.TaCommitPending));
}
/// <exception cref="System.IO.IOException"/>
public override void Done(TaskAttemptID taskId)
{
int taskIndex = this.mapIds.IndexOf(taskId);
if (taskIndex >= 0)
{
// mapping
this.status.SetMapProgress(1.0f);
}
else
{
this.status.SetReduceProgress(1.0f);
}
}
/// <summary>test without TASK_LOG_DIR</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestTaskLogWithoutTaskLogDir()
{
// TaskLog tasklog= new TaskLog();
Runtime.ClearProperty(YarnConfiguration.YarnAppContainerLogDir);
// test TaskLog
NUnit.Framework.Assert.AreEqual(TaskLog.GetMRv2LogDir(), null);
TaskAttemptID taid = Org.Mockito.Mockito.Mock <TaskAttemptID>();
JobID jid = new JobID("job", 1);
Org.Mockito.Mockito.When(((JobID)taid.GetJobID())).ThenReturn(jid);
Org.Mockito.Mockito.When(taid.ToString()).ThenReturn("JobId");
FilePath f = TaskLog.GetTaskLogFile(taid, true, TaskLog.LogName.Stdout);
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("stdout"));
}
internal static FilePath GetRealTaskLogFileLocation(TaskAttemptID taskid, bool isCleanup
, TaskLog.LogName filter)
{
TaskLog.LogFileDetail l;
try
{
l = GetLogFileDetail(taskid, filter, isCleanup);
}
catch (IOException ie)
{
Log.Error("getTaskLogFileDetail threw an exception " + ie);
return(null);
}
return(new FilePath(l.location, filter.ToString()));
}