/// <summary> /// Helper function to create the task's temporary output directory and /// return the path to the task's output file. /// </summary> /// <param name="conf">job-configuration</param> /// <param name="name">temporary task-output filename</param> /// <returns>path to the task's temporary output file</returns> /// <exception cref="System.IO.IOException"/> public static Path GetTaskOutputPath(JobConf conf, string name) { // ${mapred.out.dir} Path outputPath = GetOutputPath(conf); if (outputPath == null) { throw new IOException("Undefined job output-path"); } OutputCommitter committer = conf.GetOutputCommitter(); Path workPath = outputPath; TaskAttemptContext context = new TaskAttemptContextImpl(conf, ((TaskAttemptID)TaskAttemptID .ForName(conf.Get(JobContext.TaskAttemptId)))); if (committer is FileOutputCommitter) { workPath = ((FileOutputCommitter)committer).GetWorkPath(context, outputPath); } // ${mapred.out.dir}/_temporary/_${taskid}/${name} return(new Path(workPath, name)); }
/// <exception cref="System.Exception"/> private OutputCommitter CreateOutputCommitter(bool newApiCommitter, JobID jobId, Configuration conf) { OutputCommitter committer = null; LocalJobRunner.Log.Info("OutputCommitter set in config " + conf.Get("mapred.output.committer.class" )); if (newApiCommitter) { TaskID taskId = new TaskID(jobId, TaskType.Map, 0); TaskAttemptID taskAttemptID = new TaskAttemptID(taskId, 0); TaskAttemptContext taskContext = new TaskAttemptContextImpl(conf, taskAttemptID); OutputFormat outputFormat = ReflectionUtils.NewInstance(taskContext.GetOutputFormatClass (), conf); committer = outputFormat.GetOutputCommitter(taskContext); } else { committer = ReflectionUtils.NewInstance(conf.GetClass <OutputCommitter>("mapred.output.committer.class" , typeof(FileOutputCommitter)), conf); } LocalJobRunner.Log.Info("OutputCommitter is " + committer.GetType().FullName); return(committer); }
public override void Run() { JobID jobId = this.profile.GetJobID(); JobContext jContext = new JobContextImpl(this.job, jobId); OutputCommitter outputCommitter = null; try { outputCommitter = this.CreateOutputCommitter(this._enclosing.conf.GetUseNewMapper (), jobId, this._enclosing.conf); } catch (Exception e) { LocalJobRunner.Log.Info("Failed to createOutputCommitter", e); return; } try { JobSplit.TaskSplitMetaInfo[] taskSplitMetaInfos = SplitMetaInfoReader.ReadSplitMetaInfo (jobId, this.localFs, this._enclosing.conf, this.systemJobDir); int numReduceTasks = this.job.GetNumReduceTasks(); outputCommitter.SetupJob(jContext); this.status.SetSetupProgress(1.0f); IDictionary <TaskAttemptID, MapOutputFile> mapOutputFiles = Sharpen.Collections.SynchronizedMap (new Dictionary <TaskAttemptID, MapOutputFile>()); IList <LocalJobRunner.Job.RunnableWithThrowable> mapRunnables = this.GetMapTaskRunnables (taskSplitMetaInfos, jobId, mapOutputFiles); this.InitCounters(mapRunnables.Count, numReduceTasks); ExecutorService mapService = this.CreateMapExecutor(); this.RunTasks(mapRunnables, mapService, "map"); try { if (numReduceTasks > 0) { IList <LocalJobRunner.Job.RunnableWithThrowable> reduceRunnables = this.GetReduceTaskRunnables (jobId, mapOutputFiles); ExecutorService reduceService = this.CreateReduceExecutor(); this.RunTasks(reduceRunnables, reduceService, "reduce"); } } finally { foreach (MapOutputFile output in mapOutputFiles.Values) { output.RemoveAll(); } } // delete the temporary directory in output directory outputCommitter.CommitJob(jContext); this.status.SetCleanupProgress(1.0f); if (this.killed) { this.status.SetRunState(JobStatus.Killed); } else { this.status.SetRunState(JobStatus.Succeeded); } JobEndNotifier.LocalRunnerNotification(this.job, this.status); } catch (Exception t) { try { outputCommitter.AbortJob(jContext, JobStatus.State.Failed); } catch (IOException) { LocalJobRunner.Log.Info("Error cleaning up job:" + this.id); } this.status.SetCleanupProgress(1.0f); if (this.killed) { this.status.SetRunState(JobStatus.Killed); } else { this.status.SetRunState(JobStatus.Failed); } LocalJobRunner.Log.Warn(this.id, t); JobEndNotifier.LocalRunnerNotification(this.job, this.status); } finally { try { this._enclosing.fs.Delete(this.systemJobFile.GetParent(), true); // delete submit dir this.localFs.Delete(this.localJobFile, true); // delete local copy // Cleanup distributed cache this.localDistributedCacheManager.Close(); } catch (IOException e) { LocalJobRunner.Log.Warn("Error cleaning up " + this.id + ": " + e); } } }