/// <exception cref="System.IO.IOException"/>
public virtual bool Ping(TaskAttemptID taskAttemptID)
{
if (Log.IsDebugEnabled())
{
Log.Debug("Ping from " + taskAttemptID.ToString());
}
return(true);
}
/// <exception cref="System.IO.IOException"/>
public virtual void Done(TaskAttemptID taskAttemptID)
{
Log.Info("Done acknowledgement from " + taskAttemptID.ToString());
TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);
taskHeartbeatHandler.Progressing(attemptID);
context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType
.TaDone));
}
// TODO: This isn't really used in any MR code. Ask for removal.
/// <exception cref="System.IO.IOException"/>
public virtual MapTaskCompletionEventsUpdate GetMapCompletionEvents(JobID jobIdentifier
, int startIndex, int maxEvents, TaskAttemptID taskAttemptID)
{
Log.Info("MapCompletionEvents request from " + taskAttemptID.ToString() + ". startIndex "
+ startIndex + " maxEvents " + maxEvents);
// TODO: shouldReset is never used. See TT. Ask for Removal.
bool shouldReset = false;
TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);
TaskCompletionEvent[] events = context.GetJob(attemptID.GetTaskId().GetJobId()).GetMapAttemptCompletionEvents
(startIndex, maxEvents);
taskHeartbeatHandler.Progressing(attemptID);
return(new MapTaskCompletionEventsUpdate(events, shouldReset));
}
/// <summary>
/// TaskAttempt is reporting that it is in commit_pending and it is waiting for
/// the commit Response
/// <br />
/// Commit it a two-phased protocol.
/// </summary>
/// <remarks>
/// TaskAttempt is reporting that it is in commit_pending and it is waiting for
/// the commit Response
/// <br />
/// Commit it a two-phased protocol. First the attempt informs the
/// ApplicationMaster that it is
/// <see cref="CommitPending(TaskAttemptID, TaskStatus)"/>
/// . Then it repeatedly polls
/// the ApplicationMaster whether it
/// <see cref="CanCommit(TaskAttemptID)"/>
/// This is
/// a legacy from the centralized commit protocol handling by the JobTracker.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
public virtual void CommitPending(TaskAttemptID taskAttemptID, TaskStatus taskStatsu
)
{
Log.Info("Commit-pending state update from " + taskAttemptID.ToString());
// An attempt is asking if it can commit its output. This can be decided
// only by the task which is managing the multiple attempts. So redirect the
// request there.
TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);
taskHeartbeatHandler.Progressing(attemptID);
//Ignorable TaskStatus? - since a task will send a LastStatusUpdate
context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType
.TaCommitPending));
}
/// <summary>test without TASK_LOG_DIR</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestTaskLogWithoutTaskLogDir()
{
// TaskLog tasklog= new TaskLog();
Runtime.ClearProperty(YarnConfiguration.YarnAppContainerLogDir);
// test TaskLog
NUnit.Framework.Assert.AreEqual(TaskLog.GetMRv2LogDir(), null);
TaskAttemptID taid = Org.Mockito.Mockito.Mock <TaskAttemptID>();
JobID jid = new JobID("job", 1);
Org.Mockito.Mockito.When(((JobID)taid.GetJobID())).ThenReturn(jid);
Org.Mockito.Mockito.When(taid.ToString()).ThenReturn("JobId");
FilePath f = TaskLog.GetTaskLogFile(taid, true, TaskLog.LogName.Stdout);
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("stdout"));
}
/// <exception cref="System.IO.IOException"/>
public virtual void ReportDiagnosticInfo(TaskAttemptID taskAttemptID, string diagnosticInfo
)
{
diagnosticInfo = StringInterner.WeakIntern(diagnosticInfo);
Log.Info("Diagnostics report from " + taskAttemptID.ToString() + ": " + diagnosticInfo
);
TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);
taskHeartbeatHandler.Progressing(attemptID);
// This is mainly used for cases where we want to propagate exception traces
// of tasks that fail.
// This call exists as a hadoop mapreduce legacy wherein all changes in
// counters/progress/phase/output-size are reported through statusUpdate()
// call but not diagnosticInformation.
context.GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID,
diagnosticInfo));
}
/// <summary>test TaskAttemptID</summary>
/// <exception cref="System.IO.IOException"/>
public virtual void TestTaskLog()
{
// test TaskLog
Runtime.SetProperty(YarnConfiguration.YarnAppContainerLogDir, "testString");
NUnit.Framework.Assert.AreEqual(TaskLog.GetMRv2LogDir(), "testString");
TaskAttemptID taid = Org.Mockito.Mockito.Mock <TaskAttemptID>();
JobID jid = new JobID("job", 1);
Org.Mockito.Mockito.When(((JobID)taid.GetJobID())).ThenReturn(jid);
Org.Mockito.Mockito.When(taid.ToString()).ThenReturn("JobId");
FilePath f = TaskLog.GetTaskLogFile(taid, true, TaskLog.LogName.Stdout);
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("testString" + FilePath
.separatorChar + "stdout"));
// test getRealTaskLogFileLocation
FilePath indexFile = TaskLog.GetIndexFile(taid, true);
if (!indexFile.GetParentFile().Exists())
{
indexFile.GetParentFile().Mkdirs();
}
indexFile.Delete();
indexFile.CreateNewFile();
TaskLog.SyncLogs("location", taid, true);
NUnit.Framework.Assert.IsTrue(indexFile.GetAbsolutePath().EndsWith("userlogs" + FilePath
.separatorChar + "job_job_0001" + FilePath.separatorChar + "JobId.cleanup" + FilePath
.separatorChar + "log.index"));
f = TaskLog.GetRealTaskLogFileLocation(taid, true, TaskLog.LogName.Debugout);
if (f != null)
{
NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("location" + FilePath.
separatorChar + "debugout"));
FileUtils.CopyFile(indexFile, f);
}
// test obtainLogDirOwner
NUnit.Framework.Assert.IsTrue(TaskLog.ObtainLogDirOwner(taid).Length > 0);
// test TaskLog.Reader
NUnit.Framework.Assert.IsTrue(ReadTaskLog(TaskLog.LogName.Debugout, taid, true).Length
> 0);
}
/// <summary>Child checking whether it can commit.</summary>
/// <remarks>
/// Child checking whether it can commit.
/// <br />
/// Commit is a two-phased protocol. First the attempt informs the
/// ApplicationMaster that it is
/// <see cref="CommitPending(TaskAttemptID, TaskStatus)"/>
/// . Then it repeatedly polls
/// the ApplicationMaster whether it
/// <see cref="CanCommit(TaskAttemptID)"/>
/// This is
/// a legacy from the centralized commit protocol handling by the JobTracker.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
public virtual bool CanCommit(TaskAttemptID taskAttemptID)
{
Log.Info("Commit go/no-go request from " + taskAttemptID.ToString());
// An attempt is asking if it can commit its output. This can be decided
// only by the task which is managing the multiple attempts. So redirect the
// request there.
TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID);
taskHeartbeatHandler.Progressing(attemptID);
// tell task to retry later if AM has not heard from RM within the commit
// window to help avoid double-committing in a split-brain situation
long now = context.GetClock().GetTime();
if (now - rmHeartbeatHandler.GetLastHeartbeatTime() > commitWindowMs)
{
return(false);
}
Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(attemptID.GetTaskId
().GetJobId());
Task task = job.GetTask(attemptID.GetTaskId());
return(task.CanCommit(attemptID));
}
/// <exception cref="Sharpen.RuntimeException"/>
/// <exception cref="System.IO.IOException"/>
private void RunSubtask(Task task, TaskType taskType, TaskAttemptId attemptID, int
numMapTasks, bool renameOutputs, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles
)
{
TaskAttemptID classicAttemptID = TypeConverter.FromYarn(attemptID);
try
{
JobConf conf = new JobConf(this._enclosing.GetConfig());
conf.Set(JobContext.TaskId, task.GetTaskID().ToString());
conf.Set(JobContext.TaskAttemptId, classicAttemptID.ToString());
conf.SetBoolean(JobContext.TaskIsmap, (taskType == TaskType.Map));
conf.SetInt(JobContext.TaskPartition, task.GetPartition());
conf.Set(JobContext.Id, task.GetJobID().ToString());
// Use the AM's local dir env to generate the intermediate step
// output files
string[] localSysDirs = StringUtils.GetTrimmedStrings(Runtime.Getenv(ApplicationConstants.Environment
.LocalDirs.ToString()));
conf.SetStrings(MRConfig.LocalDir, localSysDirs);
LocalContainerLauncher.Log.Info(MRConfig.LocalDir + " for uber task: " + conf.Get
(MRConfig.LocalDir));
// mark this as an uberized subtask so it can set task counter
// (longer-term/FIXME: could redefine as job counter and send
// "JobCounterEvent" to JobImpl on [successful] completion of subtask;
// will need new Job state-machine transition and JobImpl jobCounters
// map to handle)
conf.SetBoolean("mapreduce.task.uberized", true);
// Check and handle Encrypted spill key
task.SetEncryptedSpillKey(this._enclosing.encryptedSpillKey);
YarnChild.SetEncryptedSpillKeyIfRequired(task);
// META-FIXME: do we want the extra sanity-checking (doneWithMaps,
// etc.), or just assume/hope the state machine(s) and uber-AM work
// as expected?
if (taskType == TaskType.Map)
{
if (this.doneWithMaps)
{
LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a map task ("
+ attemptID + "), but should be finished with maps");
throw new RuntimeException();
}
MapTask map = (MapTask)task;
map.SetConf(conf);
map.Run(conf, this._enclosing.umbilical);
if (renameOutputs)
{
MapOutputFile renamed = LocalContainerLauncher.RenameMapOutputForReduce(conf, attemptID
, map.GetMapOutputFile());
localMapFiles[classicAttemptID] = renamed;
}
this.Relocalize();
if (++this.finishedSubMaps == numMapTasks)
{
this.doneWithMaps = true;
}
}
else
{
/* TaskType.REDUCE */
if (!this.doneWithMaps)
{
// check if event-queue empty? whole idea of counting maps vs.
// checking event queue is a tad wacky...but could enforce ordering
// (assuming no "lost events") at LocalMRAppMaster [CURRENT BUG(?):
// doesn't send reduce event until maps all done]
LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a reduce task ("
+ attemptID + "), but not yet finished with maps");
throw new RuntimeException();
}
// a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner:
// set framework name to local to make task local
conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName);
conf.Set(MRConfig.MasterAddress, "local");
// bypass shuffle
ReduceTask reduce = (ReduceTask)task;
reduce.SetLocalMapFiles(localMapFiles);
reduce.SetConf(conf);
reduce.Run(conf, this._enclosing.umbilical);
this.Relocalize();
}
}
catch (FSError e)
{
LocalContainerLauncher.Log.Fatal("FSError from child", e);
// umbilical: MRAppMaster creates (taskAttemptListener), passes to us
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
this._enclosing.umbilical.FsError(classicAttemptID, e.Message);
}
throw new RuntimeException();
}
catch (Exception exception)
{
LocalContainerLauncher.Log.Warn("Exception running local (uberized) 'child' : " +
StringUtils.StringifyException(exception));
try
{
if (task != null)
{
// do cleanup for the task
task.TaskCleanup(this._enclosing.umbilical);
}
}
catch (Exception e)
{
LocalContainerLauncher.Log.Info("Exception cleaning up: " + StringUtils.StringifyException
(e));
}
// Report back any failures, for diagnostic purposes
this._enclosing.umbilical.ReportDiagnosticInfo(classicAttemptID, StringUtils.StringifyException
(exception));
throw new RuntimeException();
}
catch (Exception throwable)
{
LocalContainerLauncher.Log.Fatal("Error running local (uberized) 'child' : " + StringUtils
.StringifyException(throwable));
if (!ShutdownHookManager.Get().IsShutdownInProgress())
{
Exception tCause = throwable.InnerException;
string cause = (tCause == null) ? throwable.Message : StringUtils.StringifyException
(tCause);
this._enclosing.umbilical.FatalError(classicAttemptID, cause);
}
throw new RuntimeException();
}
}
public static IList <string> GetVMCommand(IPEndPoint taskAttemptListenerAddr, Task
task, JVMId jvmID)
{
TaskAttemptID attemptID = task.GetTaskID();
JobConf conf = task.conf;
Vector <string> vargs = new Vector <string>(8);
vargs.AddItem(MRApps.CrossPlatformifyMREnv(task.conf, ApplicationConstants.Environment
.JavaHome) + "/bin/java");
// Add child (task) java-vm options.
//
// The following symbols if present in mapred.{map|reduce}.child.java.opts
// value are replaced:
// + @taskid@ is interpolated with value of TaskID.
// Other occurrences of @ will not be altered.
//
// Example with multiple arguments and substitutions, showing
// jvm GC logging, and start of a passwordless JVM JMX agent so can
// connect with jconsole and the likes to watch child memory, threads
// and get thread dumps.
//
// <property>
// <name>mapred.map.child.java.opts</name>
// <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \
// -Dcom.sun.management.jmxremote.authenticate=false \
// -Dcom.sun.management.jmxremote.ssl=false \
// </value>
// </property>
//
// <property>
// <name>mapred.reduce.child.java.opts</name>
// <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \
// -Dcom.sun.management.jmxremote.authenticate=false \
// -Dcom.sun.management.jmxremote.ssl=false \
// </value>
// </property>
//
string javaOpts = GetChildJavaOpts(conf, task.IsMapTask());
javaOpts = javaOpts.Replace("@taskid@", attemptID.ToString());
string[] javaOptsSplit = javaOpts.Split(" ");
for (int i = 0; i < javaOptsSplit.Length; i++)
{
vargs.AddItem(javaOptsSplit[i]);
}
Path childTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment
.Pwd), YarnConfiguration.DefaultContainerTempDir);
vargs.AddItem("-Djava.io.tmpdir=" + childTmpDir);
MRApps.AddLog4jSystemProperties(task, vargs, conf);
if (conf.GetProfileEnabled())
{
if (conf.GetProfileTaskRange(task.IsMapTask()).IsIncluded(task.GetPartition()))
{
string profileParams = conf.Get(task.IsMapTask() ? MRJobConfig.TaskMapProfileParams
: MRJobConfig.TaskReduceProfileParams, conf.GetProfileParams());
vargs.AddItem(string.Format(profileParams, GetTaskLogFile(TaskLog.LogName.Profile
)));
}
}
// Add main class and its arguments
vargs.AddItem(typeof(YarnChild).FullName);
// main of Child
// pass TaskAttemptListener's address
vargs.AddItem(taskAttemptListenerAddr.Address.GetHostAddress());
vargs.AddItem(Sharpen.Extensions.ToString(taskAttemptListenerAddr.Port));
vargs.AddItem(attemptID.ToString());
// pass task identifier
// Finally add the jvmID
vargs.AddItem(jvmID.GetId().ToString());
vargs.AddItem("1>" + GetTaskLogFile(TaskLog.LogName.Stdout));
vargs.AddItem("2>" + GetTaskLogFile(TaskLog.LogName.Stderr));
// Final commmand
StringBuilder mergedCommand = new StringBuilder();
foreach (CharSequence str in vargs)
{
mergedCommand.Append(str).Append(" ");
}
Vector <string> vargsFinal = new Vector <string>(1);
vargsFinal.AddItem(mergedCommand.ToString());
return(vargsFinal);
}