/// <exception cref="System.IO.IOException"/> public virtual bool Ping(TaskAttemptID taskAttemptID) { if (Log.IsDebugEnabled()) { Log.Debug("Ping from " + taskAttemptID.ToString()); } return(true); }
/// <exception cref="System.IO.IOException"/> public virtual void Done(TaskAttemptID taskAttemptID) { Log.Info("Done acknowledgement from " + taskAttemptID.ToString()); TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID); taskHeartbeatHandler.Progressing(attemptID); context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType .TaDone)); }
// TODO: This isn't really used in any MR code. Ask for removal. /// <exception cref="System.IO.IOException"/> public virtual MapTaskCompletionEventsUpdate GetMapCompletionEvents(JobID jobIdentifier , int startIndex, int maxEvents, TaskAttemptID taskAttemptID) { Log.Info("MapCompletionEvents request from " + taskAttemptID.ToString() + ". startIndex " + startIndex + " maxEvents " + maxEvents); // TODO: shouldReset is never used. See TT. Ask for Removal. bool shouldReset = false; TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID); TaskCompletionEvent[] events = context.GetJob(attemptID.GetTaskId().GetJobId()).GetMapAttemptCompletionEvents (startIndex, maxEvents); taskHeartbeatHandler.Progressing(attemptID); return(new MapTaskCompletionEventsUpdate(events, shouldReset)); }
/// <summary> /// TaskAttempt is reporting that it is in commit_pending and it is waiting for /// the commit Response /// <br /> /// Commit it a two-phased protocol. /// </summary> /// <remarks> /// TaskAttempt is reporting that it is in commit_pending and it is waiting for /// the commit Response /// <br /> /// Commit it a two-phased protocol. First the attempt informs the /// ApplicationMaster that it is /// <see cref="CommitPending(TaskAttemptID, TaskStatus)"/> /// . Then it repeatedly polls /// the ApplicationMaster whether it /// <see cref="CanCommit(TaskAttemptID)"/> /// This is /// a legacy from the centralized commit protocol handling by the JobTracker. /// </remarks> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> public virtual void CommitPending(TaskAttemptID taskAttemptID, TaskStatus taskStatsu ) { Log.Info("Commit-pending state update from " + taskAttemptID.ToString()); // An attempt is asking if it can commit its output. This can be decided // only by the task which is managing the multiple attempts. So redirect the // request there. TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID); taskHeartbeatHandler.Progressing(attemptID); //Ignorable TaskStatus? - since a task will send a LastStatusUpdate context.GetEventHandler().Handle(new TaskAttemptEvent(attemptID, TaskAttemptEventType .TaCommitPending)); }
/// <summary>test without TASK_LOG_DIR</summary> /// <exception cref="System.IO.IOException"/> public virtual void TestTaskLogWithoutTaskLogDir() { // TaskLog tasklog= new TaskLog(); Runtime.ClearProperty(YarnConfiguration.YarnAppContainerLogDir); // test TaskLog NUnit.Framework.Assert.AreEqual(TaskLog.GetMRv2LogDir(), null); TaskAttemptID taid = Org.Mockito.Mockito.Mock <TaskAttemptID>(); JobID jid = new JobID("job", 1); Org.Mockito.Mockito.When(((JobID)taid.GetJobID())).ThenReturn(jid); Org.Mockito.Mockito.When(taid.ToString()).ThenReturn("JobId"); FilePath f = TaskLog.GetTaskLogFile(taid, true, TaskLog.LogName.Stdout); NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("stdout")); }
/// <exception cref="System.IO.IOException"/> public virtual void ReportDiagnosticInfo(TaskAttemptID taskAttemptID, string diagnosticInfo ) { diagnosticInfo = StringInterner.WeakIntern(diagnosticInfo); Log.Info("Diagnostics report from " + taskAttemptID.ToString() + ": " + diagnosticInfo ); TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID); taskHeartbeatHandler.Progressing(attemptID); // This is mainly used for cases where we want to propagate exception traces // of tasks that fail. // This call exists as a hadoop mapreduce legacy wherein all changes in // counters/progress/phase/output-size are reported through statusUpdate() // call but not diagnosticInformation. context.GetEventHandler().Handle(new TaskAttemptDiagnosticsUpdateEvent(attemptID, diagnosticInfo)); }
/// <summary>test TaskAttemptID</summary> /// <exception cref="System.IO.IOException"/> public virtual void TestTaskLog() { // test TaskLog Runtime.SetProperty(YarnConfiguration.YarnAppContainerLogDir, "testString"); NUnit.Framework.Assert.AreEqual(TaskLog.GetMRv2LogDir(), "testString"); TaskAttemptID taid = Org.Mockito.Mockito.Mock <TaskAttemptID>(); JobID jid = new JobID("job", 1); Org.Mockito.Mockito.When(((JobID)taid.GetJobID())).ThenReturn(jid); Org.Mockito.Mockito.When(taid.ToString()).ThenReturn("JobId"); FilePath f = TaskLog.GetTaskLogFile(taid, true, TaskLog.LogName.Stdout); NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("testString" + FilePath .separatorChar + "stdout")); // test getRealTaskLogFileLocation FilePath indexFile = TaskLog.GetIndexFile(taid, true); if (!indexFile.GetParentFile().Exists()) { indexFile.GetParentFile().Mkdirs(); } indexFile.Delete(); indexFile.CreateNewFile(); TaskLog.SyncLogs("location", taid, true); NUnit.Framework.Assert.IsTrue(indexFile.GetAbsolutePath().EndsWith("userlogs" + FilePath .separatorChar + "job_job_0001" + FilePath.separatorChar + "JobId.cleanup" + FilePath .separatorChar + "log.index")); f = TaskLog.GetRealTaskLogFileLocation(taid, true, TaskLog.LogName.Debugout); if (f != null) { NUnit.Framework.Assert.IsTrue(f.GetAbsolutePath().EndsWith("location" + FilePath. separatorChar + "debugout")); FileUtils.CopyFile(indexFile, f); } // test obtainLogDirOwner NUnit.Framework.Assert.IsTrue(TaskLog.ObtainLogDirOwner(taid).Length > 0); // test TaskLog.Reader NUnit.Framework.Assert.IsTrue(ReadTaskLog(TaskLog.LogName.Debugout, taid, true).Length > 0); }
/// <summary>Child checking whether it can commit.</summary> /// <remarks> /// Child checking whether it can commit. /// <br /> /// Commit is a two-phased protocol. First the attempt informs the /// ApplicationMaster that it is /// <see cref="CommitPending(TaskAttemptID, TaskStatus)"/> /// . Then it repeatedly polls /// the ApplicationMaster whether it /// <see cref="CanCommit(TaskAttemptID)"/> /// This is /// a legacy from the centralized commit protocol handling by the JobTracker. /// </remarks> /// <exception cref="System.IO.IOException"/> public virtual bool CanCommit(TaskAttemptID taskAttemptID) { Log.Info("Commit go/no-go request from " + taskAttemptID.ToString()); // An attempt is asking if it can commit its output. This can be decided // only by the task which is managing the multiple attempts. So redirect the // request there. TaskAttemptId attemptID = TypeConverter.ToYarn(taskAttemptID); taskHeartbeatHandler.Progressing(attemptID); // tell task to retry later if AM has not heard from RM within the commit // window to help avoid double-committing in a split-brain situation long now = context.GetClock().GetTime(); if (now - rmHeartbeatHandler.GetLastHeartbeatTime() > commitWindowMs) { return(false); } Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = context.GetJob(attemptID.GetTaskId ().GetJobId()); Task task = job.GetTask(attemptID.GetTaskId()); return(task.CanCommit(attemptID)); }
/// <exception cref="Sharpen.RuntimeException"/> /// <exception cref="System.IO.IOException"/> private void RunSubtask(Task task, TaskType taskType, TaskAttemptId attemptID, int numMapTasks, bool renameOutputs, IDictionary <TaskAttemptID, MapOutputFile> localMapFiles ) { TaskAttemptID classicAttemptID = TypeConverter.FromYarn(attemptID); try { JobConf conf = new JobConf(this._enclosing.GetConfig()); conf.Set(JobContext.TaskId, task.GetTaskID().ToString()); conf.Set(JobContext.TaskAttemptId, classicAttemptID.ToString()); conf.SetBoolean(JobContext.TaskIsmap, (taskType == TaskType.Map)); conf.SetInt(JobContext.TaskPartition, task.GetPartition()); conf.Set(JobContext.Id, task.GetJobID().ToString()); // Use the AM's local dir env to generate the intermediate step // output files string[] localSysDirs = StringUtils.GetTrimmedStrings(Runtime.Getenv(ApplicationConstants.Environment .LocalDirs.ToString())); conf.SetStrings(MRConfig.LocalDir, localSysDirs); LocalContainerLauncher.Log.Info(MRConfig.LocalDir + " for uber task: " + conf.Get (MRConfig.LocalDir)); // mark this as an uberized subtask so it can set task counter // (longer-term/FIXME: could redefine as job counter and send // "JobCounterEvent" to JobImpl on [successful] completion of subtask; // will need new Job state-machine transition and JobImpl jobCounters // map to handle) conf.SetBoolean("mapreduce.task.uberized", true); // Check and handle Encrypted spill key task.SetEncryptedSpillKey(this._enclosing.encryptedSpillKey); YarnChild.SetEncryptedSpillKeyIfRequired(task); // META-FIXME: do we want the extra sanity-checking (doneWithMaps, // etc.), or just assume/hope the state machine(s) and uber-AM work // as expected? if (taskType == TaskType.Map) { if (this.doneWithMaps) { LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a map task (" + attemptID + "), but should be finished with maps"); throw new RuntimeException(); } MapTask map = (MapTask)task; map.SetConf(conf); map.Run(conf, this._enclosing.umbilical); if (renameOutputs) { MapOutputFile renamed = LocalContainerLauncher.RenameMapOutputForReduce(conf, attemptID , map.GetMapOutputFile()); localMapFiles[classicAttemptID] = renamed; } this.Relocalize(); if (++this.finishedSubMaps == numMapTasks) { this.doneWithMaps = true; } } else { /* TaskType.REDUCE */ if (!this.doneWithMaps) { // check if event-queue empty? whole idea of counting maps vs. // checking event queue is a tad wacky...but could enforce ordering // (assuming no "lost events") at LocalMRAppMaster [CURRENT BUG(?): // doesn't send reduce event until maps all done] LocalContainerLauncher.Log.Error("CONTAINER_REMOTE_LAUNCH contains a reduce task (" + attemptID + "), but not yet finished with maps"); throw new RuntimeException(); } // a.k.a. "mapreduce.jobtracker.address" in LocalJobRunner: // set framework name to local to make task local conf.Set(MRConfig.FrameworkName, MRConfig.LocalFrameworkName); conf.Set(MRConfig.MasterAddress, "local"); // bypass shuffle ReduceTask reduce = (ReduceTask)task; reduce.SetLocalMapFiles(localMapFiles); reduce.SetConf(conf); reduce.Run(conf, this._enclosing.umbilical); this.Relocalize(); } } catch (FSError e) { LocalContainerLauncher.Log.Fatal("FSError from child", e); // umbilical: MRAppMaster creates (taskAttemptListener), passes to us if (!ShutdownHookManager.Get().IsShutdownInProgress()) { this._enclosing.umbilical.FsError(classicAttemptID, e.Message); } throw new RuntimeException(); } catch (Exception exception) { LocalContainerLauncher.Log.Warn("Exception running local (uberized) 'child' : " + StringUtils.StringifyException(exception)); try { if (task != null) { // do cleanup for the task task.TaskCleanup(this._enclosing.umbilical); } } catch (Exception e) { LocalContainerLauncher.Log.Info("Exception cleaning up: " + StringUtils.StringifyException (e)); } // Report back any failures, for diagnostic purposes this._enclosing.umbilical.ReportDiagnosticInfo(classicAttemptID, StringUtils.StringifyException (exception)); throw new RuntimeException(); } catch (Exception throwable) { LocalContainerLauncher.Log.Fatal("Error running local (uberized) 'child' : " + StringUtils .StringifyException(throwable)); if (!ShutdownHookManager.Get().IsShutdownInProgress()) { Exception tCause = throwable.InnerException; string cause = (tCause == null) ? throwable.Message : StringUtils.StringifyException (tCause); this._enclosing.umbilical.FatalError(classicAttemptID, cause); } throw new RuntimeException(); } }
public static IList <string> GetVMCommand(IPEndPoint taskAttemptListenerAddr, Task task, JVMId jvmID) { TaskAttemptID attemptID = task.GetTaskID(); JobConf conf = task.conf; Vector <string> vargs = new Vector <string>(8); vargs.AddItem(MRApps.CrossPlatformifyMREnv(task.conf, ApplicationConstants.Environment .JavaHome) + "/bin/java"); // Add child (task) java-vm options. // // The following symbols if present in mapred.{map|reduce}.child.java.opts // value are replaced: // + @taskid@ is interpolated with value of TaskID. // Other occurrences of @ will not be altered. // // Example with multiple arguments and substitutions, showing // jvm GC logging, and start of a passwordless JVM JMX agent so can // connect with jconsole and the likes to watch child memory, threads // and get thread dumps. // // <property> // <name>mapred.map.child.java.opts</name> // <value>-Xmx 512M -verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // // <property> // <name>mapred.reduce.child.java.opts</name> // <value>-Xmx 1024M -verbose:gc -Xloggc:/tmp/@[email protected] \ // -Dcom.sun.management.jmxremote.authenticate=false \ // -Dcom.sun.management.jmxremote.ssl=false \ // </value> // </property> // string javaOpts = GetChildJavaOpts(conf, task.IsMapTask()); javaOpts = javaOpts.Replace("@taskid@", attemptID.ToString()); string[] javaOptsSplit = javaOpts.Split(" "); for (int i = 0; i < javaOptsSplit.Length; i++) { vargs.AddItem(javaOptsSplit[i]); } Path childTmpDir = new Path(MRApps.CrossPlatformifyMREnv(conf, ApplicationConstants.Environment .Pwd), YarnConfiguration.DefaultContainerTempDir); vargs.AddItem("-Djava.io.tmpdir=" + childTmpDir); MRApps.AddLog4jSystemProperties(task, vargs, conf); if (conf.GetProfileEnabled()) { if (conf.GetProfileTaskRange(task.IsMapTask()).IsIncluded(task.GetPartition())) { string profileParams = conf.Get(task.IsMapTask() ? MRJobConfig.TaskMapProfileParams : MRJobConfig.TaskReduceProfileParams, conf.GetProfileParams()); vargs.AddItem(string.Format(profileParams, GetTaskLogFile(TaskLog.LogName.Profile ))); } } // Add main class and its arguments vargs.AddItem(typeof(YarnChild).FullName); // main of Child // pass TaskAttemptListener's address vargs.AddItem(taskAttemptListenerAddr.Address.GetHostAddress()); vargs.AddItem(Sharpen.Extensions.ToString(taskAttemptListenerAddr.Port)); vargs.AddItem(attemptID.ToString()); // pass task identifier // Finally add the jvmID vargs.AddItem(jvmID.GetId().ToString()); vargs.AddItem("1>" + GetTaskLogFile(TaskLog.LogName.Stdout)); vargs.AddItem("2>" + GetTaskLogFile(TaskLog.LogName.Stderr)); // Final commmand StringBuilder mergedCommand = new StringBuilder(); foreach (CharSequence str in vargs) { mergedCommand.Append(str).Append(" "); } Vector <string> vargsFinal = new Vector <string>(1); vargsFinal.AddItem(mergedCommand.ToString()); return(vargsFinal); }