//taskId should be managed as String rather than TaskID object //so that log4j can configure it from the configuration(log4j.properties). // System properties passed in from JVM runner public override void ActivateOptions() { lock (this) { SetOptionsFromSystemProperties(); if (maxEvents > 0) { tail = new List <LoggingEvent>(); } SetFile(TaskLog.GetTaskLogFile(((TaskAttemptID)TaskAttemptID.ForName(taskId)), isCleanup , TaskLog.LogName.Syslog).ToString()); SetAppend(true); base.ActivateOptions(); } }
/// <summary> /// A test that mimics a failed task to ensure that it does /// not get into the COMMIT_PENDING state, by using a fake /// UmbilicalProtocol's implementation that fails if the commit. /// </summary> /// <remarks> /// A test that mimics a failed task to ensure that it does /// not get into the COMMIT_PENDING state, by using a fake /// UmbilicalProtocol's implementation that fails if the commit. /// protocol is played. /// The test mocks the various steps in a failed task's /// life-cycle using a special OutputCommitter and UmbilicalProtocol /// implementation. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestTaskCleanupDoesNotCommit() { // Mimic a job with a special committer that does not cleanup // files when a task fails. JobConf job = new JobConf(); job.SetOutputCommitter(typeof(TestTaskCommit.CommitterWithoutCleanup)); Path outDir = new Path(rootDir, "output"); FileOutputFormat.SetOutputPath(job, outDir); // Mimic job setup string dummyAttemptID = "attempt_200707121733_0001_m_000000_0"; TaskAttemptID attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID)); OutputCommitter committer = new TestTaskCommit.CommitterWithoutCleanup(); JobContext jContext = new JobContextImpl(job, ((JobID)attemptID.GetJobID())); committer.SetupJob(jContext); // Mimic a map task dummyAttemptID = "attempt_200707121733_0001_m_000001_0"; attemptID = ((TaskAttemptID)TaskAttemptID.ForName(dummyAttemptID)); Task task = new MapTask(null, attemptID, 0, null, 1); task.SetConf(job); task.LocalizeConfiguration(job); task.Initialize(job, ((JobID)attemptID.GetJobID()), Reporter.Null, false); // Mimic the map task writing some output. string file = "test.txt"; FileSystem localFs = FileSystem.GetLocal(job); TextOutputFormat <Text, Text> theOutputFormat = new TextOutputFormat <Text, Text>(); RecordWriter <Text, Text> theRecordWriter = theOutputFormat.GetRecordWriter(localFs , job, file, Reporter.Null); theRecordWriter.Write(new Text("key"), new Text("value")); theRecordWriter.Close(Reporter.Null); // Mimic a task failure; setting up the task for cleanup simulates // the abort protocol to be played. // Without checks in the framework, this will fail // as the committer will cause a COMMIT to happen for // the cleanup task. task.SetTaskCleanupTask(); TestTaskCommit.MyUmbilical umbilical = new TestTaskCommit.MyUmbilical(this); task.Run(job, umbilical); NUnit.Framework.Assert.IsTrue("Task did not succeed", umbilical.taskDone); }
/// <summary> /// Validates map phase progress after each record is processed by map task /// using custom task reporter. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestMapProgress() { JobConf job = new JobConf(); fs = FileSystem.GetLocal(job); Path rootDir = new Path(TestRootDir); CreateInputFile(rootDir); job.SetNumReduceTasks(0); TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName("attempt_200907082313_0424_m_000000_0" )); job.SetClass("mapreduce.job.outputformat.class", typeof(NullOutputFormat), typeof( OutputFormat)); job.Set(FileInputFormat.InputDir, TestRootDir); jobId = ((JobID)taskId.GetJobID()); JobContext jContext = new JobContextImpl(job, jobId); InputFormat <object, object> input = ReflectionUtils.NewInstance(jContext.GetInputFormatClass (), job); IList <InputSplit> splits = input.GetSplits(jContext); JobSplitWriter.CreateSplitFiles(new Path(TestRootDir), job, new Path(TestRootDir) .GetFileSystem(job), splits); JobSplit.TaskSplitMetaInfo[] splitMetaInfo = SplitMetaInfoReader.ReadSplitMetaInfo (jobId, fs, job, new Path(TestRootDir)); job.SetUseNewMapper(true); // use new api for (int i = 0; i < splitMetaInfo.Length; i++) { // rawSplits.length is 1 map = new TestMapProgress.TestMapTask(this, job.Get(JTConfig.JtSystemDir, "/tmp/hadoop/mapred/system" ) + jobId + "job.xml", taskId, i, splitMetaInfo[i].GetSplitIndex(), 1); JobConf localConf = new JobConf(job); map.LocalizeConfiguration(localConf); map.SetConf(localConf); map.Run(localConf, fakeUmbilical); } // clean up fs.Delete(rootDir, true); }
public virtual void SetTaskId(string taskId) { this.SetTaskAttemptId(((TaskAttemptID)TaskAttemptID.ForName(taskId))); }
/// <summary> /// Helper function to create the task's temporary output directory and /// return the path to the task's output file. /// </summary> /// <param name="conf">job-configuration</param> /// <param name="name">temporary task-output filename</param> /// <returns>path to the task's temporary output file</returns> /// <exception cref="System.IO.IOException"/> public static Path GetTaskOutputPath(JobConf conf, string name) { // ${mapred.out.dir} Path outputPath = GetOutputPath(conf); if (outputPath == null) { throw new IOException("Undefined job output-path"); } OutputCommitter committer = conf.GetOutputCommitter(); Path workPath = outputPath; TaskAttemptContext context = new TaskAttemptContextImpl(conf, ((TaskAttemptID)TaskAttemptID .ForName(conf.Get(JobContext.TaskAttemptId)))); if (committer is FileOutputCommitter) { workPath = ((FileOutputCommitter)committer).GetWorkPath(context, outputPath); } // ${mapred.out.dir}/_temporary/_${taskid}/${name} return(new Path(workPath, name)); }
/// <exception cref="System.Exception"/> public static void Main(string[] args) { Sharpen.Thread.SetDefaultUncaughtExceptionHandler(new YarnUncaughtExceptionHandler ()); Log.Debug("Child starting"); JobConf job = new JobConf(MRJobConfig.JobConfFile); // Initing with our JobConf allows us to avoid loading confs twice Limits.Init(job); UserGroupInformation.SetConfiguration(job); string host = args[0]; int port = System.Convert.ToInt32(args[1]); IPEndPoint address = NetUtils.CreateSocketAddrForHost(host, port); TaskAttemptID firstTaskid = ((TaskAttemptID)TaskAttemptID.ForName(args[2])); long jvmIdLong = long.Parse(args[3]); JVMId jvmId = new JVMId(((JobID)firstTaskid.GetJobID()), firstTaskid.GetTaskType( ) == TaskType.Map, jvmIdLong); // initialize metrics DefaultMetricsSystem.Initialize(StringUtils.Camelize(firstTaskid.GetTaskType().ToString ()) + "Task"); // Security framework already loaded the tokens into current ugi Credentials credentials = UserGroupInformation.GetCurrentUser().GetCredentials(); Log.Info("Executing with tokens:"); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in credentials.GetAllTokens ()) { Log.Info(token); } // Create TaskUmbilicalProtocol as actual task owner. UserGroupInformation taskOwner = UserGroupInformation.CreateRemoteUser(((JobID)firstTaskid .GetJobID()).ToString()); Org.Apache.Hadoop.Security.Token.Token <JobTokenIdentifier> jt = TokenCache.GetJobToken (credentials); SecurityUtil.SetTokenService(jt, address); taskOwner.AddToken(jt); TaskUmbilicalProtocol umbilical = taskOwner.DoAs(new _PrivilegedExceptionAction_108 (address, job)); // report non-pid to application master JvmContext context = new JvmContext(jvmId, "-1000"); Log.Debug("PID: " + Sharpen.Runtime.GetEnv()["JVM_PID"]); Task task = null; UserGroupInformation childUGI = null; ScheduledExecutorService logSyncer = null; try { int idleLoopCount = 0; JvmTask myTask = null; // poll for new task for (int idle = 0; null == myTask; ++idle) { long sleepTimeMilliSecs = Math.Min(idle * 500, 1500); Log.Info("Sleeping for " + sleepTimeMilliSecs + "ms before retrying again. Got null now." ); TimeUnit.Milliseconds.Sleep(sleepTimeMilliSecs); myTask = umbilical.GetTask(context); } if (myTask.ShouldDie()) { return; } task = myTask.GetTask(); YarnChild.taskid = task.GetTaskID(); // Create the job-conf and set credentials ConfigureTask(job, task, credentials, jt); // Initiate Java VM metrics JvmMetrics.InitSingleton(jvmId.ToString(), job.GetSessionId()); childUGI = UserGroupInformation.CreateRemoteUser(Runtime.Getenv(ApplicationConstants.Environment .User.ToString())); // Add tokens to new user so that it may execute its task correctly. childUGI.AddCredentials(credentials); // set job classloader if configured before invoking the task MRApps.SetJobClassLoader(job); logSyncer = TaskLog.CreateLogSyncer(); // Create a final reference to the task for the doAs block Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_158(taskFinal, job, umbilical)); } catch (FSError e) { // use job-specified working directory // run the task Log.Fatal("FSError from child", e); if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FsError(taskid, e.Message); } } catch (Exception exception) { Log.Warn("Exception running child : " + StringUtils.StringifyException(exception) ); try { if (task != null) { // do cleanup for the task if (childUGI == null) { // no need to job into doAs block task.TaskCleanup(umbilical); } else { Task taskFinal = task; childUGI.DoAs(new _PrivilegedExceptionAction_183(taskFinal, umbilical)); } } } catch (Exception e) { Log.Info("Exception cleaning up: " + StringUtils.StringifyException(e)); } // Report back any failures, for diagnostic purposes if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { umbilical.FatalError(taskid, StringUtils.StringifyException(exception)); } } } catch (Exception throwable) { Log.Fatal("Error running child : " + StringUtils.StringifyException(throwable)); if (taskid != null) { if (!ShutdownHookManager.Get().IsShutdownInProgress()) { Exception tCause = throwable.InnerException; string cause = tCause == null ? throwable.Message : StringUtils.StringifyException (tCause); umbilical.FatalError(taskid, cause); } } } finally { RPC.StopProxy(umbilical); DefaultMetricsSystem.Shutdown(); TaskLog.SyncLogsShutdown(logSyncer); } }