/// <summary>
/// Validates map phase progress after each record is processed by map task
/// using custom task reporter.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestMapProgress()
{
JobConf job = new JobConf();
fs = FileSystem.GetLocal(job);
Path rootDir = new Path(TestRootDir);
CreateInputFile(rootDir);
job.SetNumReduceTasks(0);
TaskAttemptID taskId = ((TaskAttemptID)TaskAttemptID.ForName("attempt_200907082313_0424_m_000000_0"
));
job.SetClass("mapreduce.job.outputformat.class", typeof(NullOutputFormat), typeof(
OutputFormat));
job.Set(FileInputFormat.InputDir, TestRootDir);
jobId = ((JobID)taskId.GetJobID());
JobContext jContext = new JobContextImpl(job, jobId);
InputFormat <object, object> input = ReflectionUtils.NewInstance(jContext.GetInputFormatClass
(), job);
IList <InputSplit> splits = input.GetSplits(jContext);
JobSplitWriter.CreateSplitFiles(new Path(TestRootDir), job, new Path(TestRootDir)
.GetFileSystem(job), splits);
JobSplit.TaskSplitMetaInfo[] splitMetaInfo = SplitMetaInfoReader.ReadSplitMetaInfo
(jobId, fs, job, new Path(TestRootDir));
job.SetUseNewMapper(true);
// use new api
for (int i = 0; i < splitMetaInfo.Length; i++)
{
// rawSplits.length is 1
map = new TestMapProgress.TestMapTask(this, job.Get(JTConfig.JtSystemDir, "/tmp/hadoop/mapred/system"
) + jobId + "job.xml", taskId, i, splitMetaInfo[i].GetSplitIndex(), 1);
JobConf localConf = new JobConf(job);
map.LocalizeConfiguration(localConf);
map.SetConf(localConf);
map.Run(localConf, fakeUmbilical);
}
// clean up
fs.Delete(rootDir, true);
}
public override void Run()
{
JobID jobId = this.profile.GetJobID();
JobContext jContext = new JobContextImpl(this.job, jobId);
OutputCommitter outputCommitter = null;
try
{
outputCommitter = this.CreateOutputCommitter(this._enclosing.conf.GetUseNewMapper
(), jobId, this._enclosing.conf);
}
catch (Exception e)
{
LocalJobRunner.Log.Info("Failed to createOutputCommitter", e);
return;
}
try
{
JobSplit.TaskSplitMetaInfo[] taskSplitMetaInfos = SplitMetaInfoReader.ReadSplitMetaInfo
(jobId, this.localFs, this._enclosing.conf, this.systemJobDir);
int numReduceTasks = this.job.GetNumReduceTasks();
outputCommitter.SetupJob(jContext);
this.status.SetSetupProgress(1.0f);
IDictionary <TaskAttemptID, MapOutputFile> mapOutputFiles = Sharpen.Collections.SynchronizedMap
(new Dictionary <TaskAttemptID, MapOutputFile>());
IList <LocalJobRunner.Job.RunnableWithThrowable> mapRunnables = this.GetMapTaskRunnables
(taskSplitMetaInfos, jobId, mapOutputFiles);
this.InitCounters(mapRunnables.Count, numReduceTasks);
ExecutorService mapService = this.CreateMapExecutor();
this.RunTasks(mapRunnables, mapService, "map");
try
{
if (numReduceTasks > 0)
{
IList <LocalJobRunner.Job.RunnableWithThrowable> reduceRunnables = this.GetReduceTaskRunnables
(jobId, mapOutputFiles);
ExecutorService reduceService = this.CreateReduceExecutor();
this.RunTasks(reduceRunnables, reduceService, "reduce");
}
}
finally
{
foreach (MapOutputFile output in mapOutputFiles.Values)
{
output.RemoveAll();
}
}
// delete the temporary directory in output directory
outputCommitter.CommitJob(jContext);
this.status.SetCleanupProgress(1.0f);
if (this.killed)
{
this.status.SetRunState(JobStatus.Killed);
}
else
{
this.status.SetRunState(JobStatus.Succeeded);
}
JobEndNotifier.LocalRunnerNotification(this.job, this.status);
}
catch (Exception t)
{
try
{
outputCommitter.AbortJob(jContext, JobStatus.State.Failed);
}
catch (IOException)
{
LocalJobRunner.Log.Info("Error cleaning up job:" + this.id);
}
this.status.SetCleanupProgress(1.0f);
if (this.killed)
{
this.status.SetRunState(JobStatus.Killed);
}
else
{
this.status.SetRunState(JobStatus.Failed);
}
LocalJobRunner.Log.Warn(this.id, t);
JobEndNotifier.LocalRunnerNotification(this.job, this.status);
}
finally
{
try
{
this._enclosing.fs.Delete(this.systemJobFile.GetParent(), true);
// delete submit dir
this.localFs.Delete(this.localJobFile, true);
// delete local copy
// Cleanup distributed cache
this.localDistributedCacheManager.Close();
}
catch (IOException e)
{
LocalJobRunner.Log.Warn("Error cleaning up " + this.id + ": " + e);
}
}
}
