/// <summary>
/// Helper function to create the task's temporary output directory and
/// return the path to the task's output file.
/// </summary>
/// <param name="conf">job-configuration</param>
/// <param name="name">temporary task-output filename</param>
/// <returns>path to the task's temporary output file</returns>
/// <exception cref="System.IO.IOException"/>
public static Path GetTaskOutputPath(JobConf conf, string name)
{
// ${mapred.out.dir}
Path outputPath = GetOutputPath(conf);
if (outputPath == null)
{
throw new IOException("Undefined job output-path");
}
OutputCommitter committer = conf.GetOutputCommitter();
Path workPath = outputPath;
TaskAttemptContext context = new TaskAttemptContextImpl(conf, ((TaskAttemptID)TaskAttemptID
.ForName(conf.Get(JobContext.TaskAttemptId))));
if (committer is FileOutputCommitter)
{
workPath = ((FileOutputCommitter)committer).GetWorkPath(context, outputPath);
}
// ${mapred.out.dir}/_temporary/_${taskid}/${name}
return(new Path(workPath, name));
}
/// <exception cref="System.Exception"/>
private OutputCommitter CreateOutputCommitter(bool newApiCommitter, JobID jobId,
Configuration conf)
{
OutputCommitter committer = null;
LocalJobRunner.Log.Info("OutputCommitter set in config " + conf.Get("mapred.output.committer.class"
));
if (newApiCommitter)
{
TaskID taskId = new TaskID(jobId, TaskType.Map, 0);
TaskAttemptID taskAttemptID = new TaskAttemptID(taskId, 0);
TaskAttemptContext taskContext = new TaskAttemptContextImpl(conf, taskAttemptID);
OutputFormat outputFormat = ReflectionUtils.NewInstance(taskContext.GetOutputFormatClass
(), conf);
committer = outputFormat.GetOutputCommitter(taskContext);
}
else
{
committer = ReflectionUtils.NewInstance(conf.GetClass <OutputCommitter>("mapred.output.committer.class"
, typeof(FileOutputCommitter)), conf);
}
LocalJobRunner.Log.Info("OutputCommitter is " + committer.GetType().FullName);
return(committer);
}
public override void Run()
{
JobID jobId = this.profile.GetJobID();
JobContext jContext = new JobContextImpl(this.job, jobId);
OutputCommitter outputCommitter = null;
try
{
outputCommitter = this.CreateOutputCommitter(this._enclosing.conf.GetUseNewMapper
(), jobId, this._enclosing.conf);
}
catch (Exception e)
{
LocalJobRunner.Log.Info("Failed to createOutputCommitter", e);
return;
}
try
{
JobSplit.TaskSplitMetaInfo[] taskSplitMetaInfos = SplitMetaInfoReader.ReadSplitMetaInfo
(jobId, this.localFs, this._enclosing.conf, this.systemJobDir);
int numReduceTasks = this.job.GetNumReduceTasks();
outputCommitter.SetupJob(jContext);
this.status.SetSetupProgress(1.0f);
IDictionary <TaskAttemptID, MapOutputFile> mapOutputFiles = Sharpen.Collections.SynchronizedMap
(new Dictionary <TaskAttemptID, MapOutputFile>());
IList <LocalJobRunner.Job.RunnableWithThrowable> mapRunnables = this.GetMapTaskRunnables
(taskSplitMetaInfos, jobId, mapOutputFiles);
this.InitCounters(mapRunnables.Count, numReduceTasks);
ExecutorService mapService = this.CreateMapExecutor();
this.RunTasks(mapRunnables, mapService, "map");
try
{
if (numReduceTasks > 0)
{
IList <LocalJobRunner.Job.RunnableWithThrowable> reduceRunnables = this.GetReduceTaskRunnables
(jobId, mapOutputFiles);
ExecutorService reduceService = this.CreateReduceExecutor();
this.RunTasks(reduceRunnables, reduceService, "reduce");
}
}
finally
{
foreach (MapOutputFile output in mapOutputFiles.Values)
{
output.RemoveAll();
}
}
// delete the temporary directory in output directory
outputCommitter.CommitJob(jContext);
this.status.SetCleanupProgress(1.0f);
if (this.killed)
{
this.status.SetRunState(JobStatus.Killed);
}
else
{
this.status.SetRunState(JobStatus.Succeeded);
}
JobEndNotifier.LocalRunnerNotification(this.job, this.status);
}
catch (Exception t)
{
try
{
outputCommitter.AbortJob(jContext, JobStatus.State.Failed);
}
catch (IOException)
{
LocalJobRunner.Log.Info("Error cleaning up job:" + this.id);
}
this.status.SetCleanupProgress(1.0f);
if (this.killed)
{
this.status.SetRunState(JobStatus.Killed);
}
else
{
this.status.SetRunState(JobStatus.Failed);
}
LocalJobRunner.Log.Warn(this.id, t);
JobEndNotifier.LocalRunnerNotification(this.job, this.status);
}
finally
{
try
{
this._enclosing.fs.Delete(this.systemJobFile.GetParent(), true);
// delete submit dir
this.localFs.Delete(this.localJobFile, true);
// delete local copy
// Cleanup distributed cache
this.localDistributedCacheManager.Close();
}
catch (IOException e)
{
LocalJobRunner.Log.Warn("Error cleaning up " + this.id + ": " + e);
}
}
}
